[#873] Give each backend its own compressed schema trees

opendj-server-legacy/src/main/java/org/opends/server/backends/cassandra/CASStorage.java

@@ -27,6 +27,7 @@
 import java.util.NoSuchElementException;
 import java.util.Objects;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import org.forgerock.i18n.LocalizableMessage;
 import org.forgerock.i18n.slf4j.LocalizedLogger;
@@ -63,6 +64,7 @@
 import com.datastax.oss.driver.api.core.cql.ResultSet;
 import com.datastax.oss.driver.api.core.cql.Row;
 import com.datastax.oss.driver.api.core.cql.Statement;
+import com.datastax.oss.driver.api.core.servererrors.InvalidQueryException;
 
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.benmanes.caffeine.cache.LoadingCache;
@@ -204,6 +206,13 @@ public void write(WriteOperation writeOperation) throws Exception {
 			System.setProperty("datastax-java-driver.profiles."+profile+".basic.request.timeout", "30 seconds");
 		}
 	}
+	// The wordings a server uses for a table or a keyspace that is not there. Matched rather than
+	// the exception type, which covers the whole INVALID protocol code: see namesAnAbsentTable().
+	// "Undefined column name ..." deliberately does not match - that table exists.
+	static final Pattern ABSENT_TABLE=Pattern.compile(
+		"unconfigured (table|columnfamily)|(table|keyspace)[^,]* does not exist|undefined (table|keyspace)",
+		Pattern.CASE_INSENSITIVE);
+
 	private final class TransactionImpl implements ReadableTransaction,WriteableTransaction {
 
 		final AccessMode accessMode;
@@ -247,6 +256,57 @@ public long getRecordCount(TreeName treeName) {
 			).one().getLong(0);
 		}
 
+		@Override
+		public boolean treeExists(TreeName treeName) {
+			// Every tree of this backend is a partition of the one table named after the backend id,
+			// so a tree has no existence apart from its rows: "exists" here means "holds at least one
+			// record". A LIMIT 1 lookup rather than getRecordCount() to avoid the full partition scan
+			// a count would run.
+			try {
+				return execute(
+					prepared.get("SELECT key FROM "+getTableName()+" WHERE baseDN=:baseDN and indexId=:indexId LIMIT 1").bind()
+						.setString("baseDN", treeName.getBaseDN()).setString("indexId", treeName.getIndexId())
+				).one()!=null;
+			}catch (InvalidQueryException e) {
+				// The backend's own table has not been created yet - a read-only open of a backend
+				// that was never written, where openTree() creates nothing - so none of its trees
+				// can exist either. The driver reports it from prepare() as much as from execute().
+				//
+				// Narrowly, twice over, because calling a populated tree absent would have the
+				// compressed schema restart its token allocation from zero and overwrite the
+				// definitions the entries already written were encoded with (#873):
+				//
+				// - InvalidQueryException carries the whole INVALID protocol code, so the rejection
+				//   has to name an absent table or keyspace. An undefined column, say, means the
+				//   table is there with a shape this backend did not write, and answering "absent"
+				//   for it would be that same corruption;
+				// - and only a read-only open may answer it at all. A writeable open has just run
+				//   CREATE TABLE IF NOT EXISTS through openTree(), so a rejection there is a fault
+				//   and must fail the open, as it did before this method existed and loadTrees()
+				//   opened its cursor unconditionally. This is also where a table a coordinator has
+				//   not caught up with lands - what a rolling upgrade produces, since schema
+				//   agreement is never reached in a mixed-version cluster - and it fails loudly
+				//   rather than being taken for a table that was never created.
+				if (accessMode.isWriteable() || !namesAnAbsentTable(e)) {
+					throw e;
+				}
+				return false;
+			}
+		}
+
+		/**
+		 * Whether a rejected query says that the table or the keyspace is not there, as opposed to
+		 * anything else the INVALID protocol code covers. The driver offers nothing finer than the
+		 * server's own message - the code is one value for the whole bucket - so the wordings of
+		 * the server are matched, across the versions that changed them ("unconfigured
+		 * columnfamily" became "unconfigured table", and a keyspace is reported as not existing).
+		 * A wording that is not among them is not read as an absent table: it fails the open, which
+		 * is the side to err on.
+		 */
+		private boolean namesAnAbsentTable(InvalidQueryException e) {
+			return e.getMessage()!=null && ABSENT_TABLE.matcher(e.getMessage()).find();
+		}
+
 		@Override
 		public void deleteTree(TreeName treeName) {
 			checkReadOnly();

opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/JDBCStorage.java

@@ -205,27 +205,62 @@ public void close() {
 		unstampableTrees.clear();
 	}
 
+	// The trees this storage has taken an interest in, and the tables they map to. listTrees() -
+	// and through it removeStorageFiles() - reads this, so a tree only belongs here once this
+	// backend uses it: see toTableName() below for the trees that are merely asked about.
 	final LoadingCache<TreeName,String> tree2table = Caffeine.newBuilder()
-		.build(treeName -> {
-			try {
-				final MessageDigest md = MessageDigest.getInstance("SHA-224");
-				final byte[] messageDigest = md.digest(treeName.toString().getBytes());
-				final StringBuilder hashtext = new StringBuilder(56);
-				for (byte b : messageDigest) {
-					String hex = Integer.toHexString(0xff & b);
-					if (hex.length() == 1) hashtext.append('0');
-					hashtext.append(hex);
-				}
-				return "opendj_" + hashtext;
-			} catch (NoSuchAlgorithmException e) {
-				throw new RuntimeException(e);
-			}
-		});
+		.build(JDBCStorage::toTableName);
+
+	/**
+	 * The table a tree name maps to. A pure function of the name, so that a tree can be read
+	 * without being entered into tree2table: the compressed schema reads the tree its definitions
+	 * used to be shared under (#873), a tree this backend does not own, and removeStorageFiles()
+	 * drops every table tree2table names.
+	 * <p>
+	 * Which of the two a statement takes therefore says who owns the tree it names: a path that
+	 * creates or writes one - openTree(), clearTree(), deleteTree(), put(), update(), delete() -
+	 * takes the enrolling {@link #getTableName(TreeName)}, and a read-only path - read(),
+	 * getRecordCount(), isExistsTable() and the cursor - takes {@link #readTableName(TreeName)},
+	 * which computes this only for a tree that is not enrolled already. Every tree this backend
+	 * owns passes through openTree(name, true) as it is opened, so listTrees() still names the
+	 * complete owned set.
+	 */
+	static String toTableName(TreeName treeName) {
+		try {
+			final MessageDigest md = MessageDigest.getInstance("SHA-224");
+			final byte[] messageDigest = md.digest(treeName.toString().getBytes());
+			final StringBuilder hashtext = new StringBuilder(56);
+			for (byte b : messageDigest) {
+				String hex = Integer.toHexString(0xff & b);
+				if (hex.length() == 1) hashtext.append('0');
+				hashtext.append(hex);
+			}
+			return "opendj_" + hashtext;
+		} catch (NoSuchAlgorithmException e) {
+			throw new RuntimeException(e);
+		}
+	}
 
 	String getTableName(TreeName treeName) {
 		return tree2table.get(treeName);
 	}
 
+	/**
+	 * The table a tree name maps to, for a statement that only reads it. Answered from the memo of
+	 * {@link #getTableName(TreeName)} where the tree is in it, and computed without being put there
+	 * otherwise.
+	 * <p>
+	 * Every tree this backend owns is enrolled as it is opened, so the per-entry read path stays a
+	 * map lookup: {@link #toTableName(TreeName)} takes a JCA provider lookup and a digest per call,
+	 * which read() would otherwise pay for every entry of every search. Only a tree this backend
+	 * does not own - the shared compressed schema tree the migration of #873 reads - is computed,
+	 * twice per open of the backend.
+	 */
+	String readTableName(TreeName treeName) {
+		final String enrolled=tree2table.getIfPresent(treeName);
+		return enrolled!=null ? enrolled : toTableName(treeName);
+	}
+
 	/**
 	 * The form a catalog pattern has to take to match an identifier this backend created unquoted.
 	 * An unquoted identifier is folded when it is stored - to upper case on oracle, to lower case
@@ -1305,7 +1340,7 @@ public ReadableTransactionImpl(Connection con) {
 
 		@Override
 		public ByteString read(TreeName treeName, ByteSequence key) {
-			try (final PreparedStatement statement=con.prepareStatement("select v from "+getTableName(treeName)+" where h="+hashParam(con)+" and k=?")){
+			try (final PreparedStatement statement=con.prepareStatement("select v from "+readTableName(treeName)+" where h="+hashParam(con)+" and k=?")){
 				statement.setString(1,key2hash.get(ByteBuffer.wrap(key.toByteArray())));
 				statement.setBytes(2,real2db(key.toByteArray()));
 				try(ResultSet rc=executeResultSet(statement)) {
@@ -1323,13 +1358,47 @@ public Cursor<ByteString, ByteString> openCursor(TreeName treeName) {
 
 		@Override
 		public long getRecordCount(TreeName treeName) {
-			try (final PreparedStatement statement=con.prepareStatement("select count(*) from "+getTableName(treeName));
+			try (final PreparedStatement statement=con.prepareStatement("select count(*) from "+readTableName(treeName));
 				 final ResultSet rc=executeResultSet(statement)){
 				return rc.next() ? rc.getLong(1) : 0;
 			}catch (SQLException e) {
 				throw new StorageRuntimeException(e);
 			}
 		}
+
+		@Override
+		public boolean treeExists(TreeName treeName) {
+			return isExistsTable(treeName);
+		}
+
+		// Readable, not writeable: the caller that asks about a tree this backend does not own is
+		// the compressed schema migration (#873), which probes the shared tree from the writeable
+		// transaction of RootContainer.open() but must not create or enrol it. Answering that from
+		// the readable transaction keeps the probe available to every reader, and costs nothing:
+		// the writeable one inherits it.
+		boolean isExistsTable(TreeName treeName) {
+			final String tableName = readTableName(treeName);
+			try {
+				final DatabaseMetaData metaData = con.getMetaData();
+				// asked of the catalog by name: openTree(createOnDemand) calls this for every tree
+				// of the backend - about 25 of them for a stock suffix, on every open - and listing
+				// every table of the database each time costs the whole catalog once per tree, on a
+				// database this backend may well be sharing with something else
+				try (final ResultSet rs = metaData.getTables(null, null,
+						storedIdentifier(metaData, tableName), new String[]{"TABLE"})) {
+					while (rs.next()) {
+						// the name still has to be compared: "_" is a single-character wildcard in a
+						// metadata pattern, so "opendj_<hash>" also matches a table named "opendjX<hash>"
+						if (tableName.equalsIgnoreCase(rs.getString("TABLE_NAME"))) {
+							return true;
+						}
+					}
+				}
+			} catch (Exception e) {
+				throw new StorageRuntimeException(e);
+			}
+			return false;
+		}
 	}
 	/**
 	 * A transaction able to write, unless the storage was opened read-only: then it may open an existing tree and
@@ -1405,30 +1474,6 @@ private boolean commitsBeforeDdl() {
 			return driverName.contains("mysql") || driverName.contains("oracle");
 		}
 
-		boolean isExistsTable(TreeName treeName) {
-			final String tableName = getTableName(treeName);
-			try {
-				final DatabaseMetaData metaData = con.getMetaData();
-				// asked of the catalog by name: openTree(createOnDemand) calls this for every tree
-				// of the backend - about 25 of them for a stock suffix, on every open - and listing
-				// every table of the database each time costs the whole catalog once per tree, on a
-				// database this backend may well be sharing with something else
-				try (final ResultSet rs = metaData.getTables(null, null,
-						storedIdentifier(metaData, tableName), new String[]{"TABLE"})) {
-					while (rs.next()) {
-						// the name still has to be compared: "_" is a single-character wildcard in a
-						// metadata pattern, so "opendj_<hash>" also matches a table named "opendjX<hash>"
-						if (tableName.equalsIgnoreCase(rs.getString("TABLE_NAME"))) {
-							return true;
-						}
-					}
-				}
-			} catch (Exception e) {
-				throw new StorageRuntimeException(e);
-			}
-			return false;
-		}
-
 		String getTableDialect() {
 			if (driverNameOf(con).contains("oracle")) {
 				return "h char(128),k raw(2000),v blob,primary key(h,k)";
@@ -1645,7 +1690,10 @@ static int compareKeys(byte[] key1, byte[] key2) {
 	// repositioning transfer "fetchsize" rows over the network (#860).
 	final class CursorImpl implements Cursor<ByteString, ByteString> {
 		final Connection con;
+		final TreeName treeName;
 		final String tableName;
+		// the enrolling name, resolved once and only if this cursor ever deletes
+		String writeTableName;
 		final boolean isReadOnly;
 		final int batchSize=Math.max(1,Integer.getInteger("org.openidentityplatform.opendj.jdbc.fetchsize",1000));
 		final int initialBatchSize=Math.min(batchSize,Math.max(1,Integer.getInteger("org.openidentityplatform.opendj.jdbc.fetchsize.initial",32)));
@@ -1662,7 +1710,10 @@ final class CursorImpl implements Cursor<ByteString, ByteString> {
 		public CursorImpl(boolean isReadOnly, Connection con, TreeName treeName) {
 			this.isReadOnly=isReadOnly;
 			this.con=con;
-			this.tableName=getTableName(treeName);
+			this.treeName=treeName;
+			// the read statements below take the non-enrolling name: a cursor is how the migration
+			// of #873 reads the shared tree, and reading a tree must not put it up for removal
+			this.tableName=readTableName(treeName);
 			this.limitClause=((CachedConnection)con).parent.getClass().getName().contains("mysql")
 				? " limit ?,?" : " offset ? rows fetch next ? rows only";
 		}
@@ -1743,7 +1794,12 @@ public void delete() throws NoSuchElementException, UnsupportedOperationExceptio
 			if (isReadOnly) {
 				throw new UnsupportedOperationException();
 			}
-			try (final PreparedStatement statement=con.prepareStatement("delete from "+tableName+" where h="+hashParam(con)+" and k=?")){
+			if (writeTableName==null) {
+				// the enrolling name, unlike the read statements above: this writes to the tree, so
+				// it is one this backend owns, and removeStorageFiles() has to know about it
+				writeTableName=getTableName(treeName);
+			}
+			try (final PreparedStatement statement=con.prepareStatement("delete from "+writeTableName+" where h="+hashParam(con)+" and k=?")){
 				statement.setString(1,key2hash.get(ByteBuffer.wrap(db2real(currentKeyDb))));
 				statement.setBytes(2,currentKeyDb);
 				execute(statement);

opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/JEStorage.java

@@ -427,6 +427,21 @@ public long getRecordCount(TreeName treeName)
       }
     }
 
+    @Override
+    public boolean treeExists(TreeName treeName)
+    {
+      // Deliberately not getOrOpenTree(): dbConfig() sets allowCreate, so asking the tree itself
+      // would create the very database whose absence is being tested.
+      try
+      {
+        return env.getDatabaseNames().contains(toDatabaseName(treeName));
+      }
+      catch (DatabaseException e)
+      {
+        throw new StorageRuntimeException(e);
+      }
+    }
+
     @Override
     public Cursor<ByteString, ByteString> openCursor(final TreeName treeName)
     {
@@ -550,6 +565,12 @@ public long getRecordCount(TreeName treeName)
       return delegate.getRecordCount(treeName);
     }
 
+    @Override
+    public boolean treeExists(TreeName treeName)
+    {
+      return delegate.treeExists(treeName);
+    }
+
     @Override
     public void openTree(TreeName treeName, boolean createOnDemand)
     {
@@ -638,6 +659,13 @@ public long getRecordCount(TreeName treeName)
     {
       return 0;
     }
+
+    @Override
+    public boolean treeExists(TreeName treeName)
+    {
+      // No environment was ever opened, so nothing is stored.
+      return false;
+    }
   }
 
   private WriteableTransaction newWriteableTransaction(Transaction txn)