From eb62ed6d8ef6c01c76861a7194ce6fdd9aac8b07 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 3 Mar 2026 16:14:32 +0200 Subject: [PATCH 01/93] WIP: Implement models for incremental reprocessing. --- .../src/storage/MongoBucketStorage.ts | 9 + .../implementation/BucketDefinitionMapping.ts | 57 +++++ .../implementation/MongoBucketBatch.ts | 95 +++++--- .../implementation/MongoPersistedSyncRules.ts | 62 +++++ .../MongoPersistedSyncRulesContent.ts | 19 +- .../implementation/MongoSyncBucketStorage.ts | 33 ++- .../storage/implementation/PersistedBatch.ts | 222 +++++++++++++----- .../src/storage/implementation/db.ts | 63 ++++- .../src/storage/implementation/models.ts | 51 +++- .../src/storage/storage-index.ts | 2 + .../test/src/storage_sync.test.ts | 39 ++- 11 files changed, 551 insertions(+), 101 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 90ba0d945..8cfbdb106 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -2,6 +2,7 @@ import { GetIntanceOptions, storage } from '@powersync/service-core'; import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { v4 as uuid } from 'uuid'; +import { SqlSyncRules } from '@powersync/service-sync-rules'; import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; @@ -12,6 +13,7 @@ import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedS import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; import { generateSlotName } from '../utils/util.js'; import { MongoChecksumOptions } from './implementation/MongoChecksums.js'; +import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; export interface MongoBucketStorageOptions { checksumOptions?: Omit; @@ -203,6 +205,13 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { last_fatal_error_ts: null, last_keepalive_ts: null }; + if (storageConfig.incrementalReprocessing) { + const parsed = SqlSyncRules.fromYaml(options.config.yaml, { + schema: undefined, + defaultSchema: 'not_applicable' + }); + doc.rule_mapping = BucketDefinitionMapping.fromParsedSyncRules(parsed).serialize(); + } await this.db.sync_rules.insertOne(doc); await this.db.notifyCheckpoint(); rules = new MongoPersistedSyncRulesContent(this.db, doc); diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts new file mode 100644 index 000000000..f066a2013 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -0,0 +1,57 @@ +import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { BucketDataSource, ParameterIndexLookupCreator, SyncConfigWithErrors } from '@powersync/service-sync-rules'; +import { SyncRuleDocument } from './models.js'; + +export class BucketDefinitionMapping { + static fromSyncRules(doc: Pick): BucketDefinitionMapping { + return new BucketDefinitionMapping(doc.rule_mapping?.definitions ?? {}, doc.rule_mapping?.parameter_lookups ?? {}); + } + + static fromParsedSyncRules(syncRules: SyncConfigWithErrors): BucketDefinitionMapping { + const definitionNames = syncRules.config.bucketDataSources.map((source) => source.uniqueName).sort(); + const parameterKeys = syncRules.config.bucketParameterLookupSources + .map((source) => `${source.defaultLookupScope.lookupName}#${source.defaultLookupScope.queryId}`) + .sort(); + + const definitions: Record = {}; + const parameterLookups: Record = {}; + + for (const [index, uniqueName] of definitionNames.entries()) { + definitions[uniqueName] = index + 1; + } + for (const [index, key] of parameterKeys.entries()) { + parameterLookups[key] = index + 1; + } + + return new BucketDefinitionMapping(definitions, parameterLookups); + } + + constructor( + private definitions: Record = {}, + private parameterLookupMapping: Record = {} + ) {} + + bucketSourceId(source: BucketDataSource): number { + const defId = this.definitions[source.uniqueName]; + if (defId == null) { + throw new ServiceAssertionError(`No mapping found for bucket source ${source.uniqueName}`); + } + return defId; + } + + parameterLookupId(source: ParameterIndexLookupCreator): number { + const key = `${source.defaultLookupScope.lookupName}#${source.defaultLookupScope.queryId}`; + const defId = this.parameterLookupMapping[key]; + if (defId == null) { + throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`); + } + return defId; + } + + serialize(): NonNullable { + return { + definitions: { ...this.definitions }, + parameter_lookups: { ...this.parameterLookupMapping } + }; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index ffad6e036..4afaf8f98 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -25,12 +25,21 @@ import { } from '@powersync/service-core'; import * as timers from 'node:timers/promises'; import { idPrefixFilter, mongoTableId } from '../../utils/util.js'; -import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js'; -import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js'; +import { VersionedPowerSyncMongo } from './db.js'; +import { + CommonCurrentBucket, + CommonCurrentLookup, + CommonCurrentDataDocument, + CurrentBucketV3, + RecordedLookupV3, + SourceKey, + SyncRuleDocument +} from './models.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; /** * 15MB @@ -55,6 +64,7 @@ export interface MongoBucketBatchOptions { keepaliveOp: InternalOpId | null; resumeFromLsn: string | null; storeCurrentData: boolean; + mapping: BucketDefinitionMapping; /** * Set to true for initial replication. */ @@ -81,6 +91,7 @@ export class MongoBucketBatch private readonly slot_name: string; private readonly storeCurrentData: boolean; private readonly skipExistingRows: boolean; + private readonly mapping: BucketDefinitionMapping; private batch: OperationBatch | null = null; private write_checkpoint_batch: storage.CustomWriteCheckpointOptions[] = []; @@ -129,6 +140,7 @@ export class MongoBucketBatch this.slot_name = options.slotName; this.sync_rules = options.syncRules; this.storeCurrentData = options.storeCurrentData; + this.mapping = options.mapping; this.skipExistingRows = options.skipExistingRows; this.markRecordUnavailable = options.markRecordUnavailable; this.batch = new OperationBatch(); @@ -259,7 +271,7 @@ export class MongoBucketBatch const lookups: SourceKey[] = b.map((r) => { return { g: this.group_id, t: mongoTableId(r.record.sourceTable.id), k: r.beforeId }; }); - let current_data_lookup = new Map(); + let current_data_lookup = new Map(); // With skipExistingRows, we only need to know whether or not the row exists. const projection = this.skipExistingRows ? { _id: 1 } : undefined; const cursor = this.db.common_current_data.find( @@ -272,9 +284,15 @@ export class MongoBucketBatch current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc); } - let persistedBatch: PersistedBatch | null = new PersistedBatch(this.db, this.group_id, transactionSize, { - logger: this.logger - }); + let persistedBatch: PersistedBatch | null = new PersistedBatch( + this.db, + this.group_id, + this.mapping, + transactionSize, + { + logger: this.logger + } + ); for (let op of b) { if (resumeBatch) { @@ -323,7 +341,7 @@ export class MongoBucketBatch private saveOperation( batch: PersistedBatch, operation: RecordOperation, - current_data: CurrentDataDocument | null, + current_data: CommonCurrentDataDocument | null, opSeq: MongoIdSequence ) { const record = operation.record; @@ -332,10 +350,10 @@ export class MongoBucketBatch let after = record.after; const sourceTable = record.sourceTable; - let existing_buckets: CurrentBucket[] = []; - let new_buckets: CurrentBucket[] = []; - let existing_lookups: bson.Binary[] = []; - let new_lookups: bson.Binary[] = []; + let existing_buckets: CommonCurrentBucket[] = []; + let new_buckets: CommonCurrentBucket[] = []; + let existing_lookups: CommonCurrentLookup[] = []; + let new_lookups: CommonCurrentLookup[] = []; const before_key: SourceKey = { g: this.group_id, t: mongoTableId(record.sourceTable.id), k: beforeId }; @@ -498,13 +516,25 @@ export class MongoBucketBatch table: sourceTable, before_buckets: existing_buckets }); - new_buckets = evaluated.map((e) => { - return { - bucket: e.bucket, - table: e.table, - id: e.id - }; - }); + if (this.db.storageConfig.incrementalReprocessing) { + new_buckets = evaluated.map((e) => { + const def = this.mapping.bucketSourceId(e.source); + return { + def, + bucket: e.bucket, + table: e.table, + id: e.id + } satisfies CurrentBucketV3; + }); + } else { + new_buckets = evaluated.map((e) => { + return { + bucket: e.bucket, + table: e.table, + id: e.id + }; + }); + } } if (sourceTable.syncParameters) { @@ -537,19 +567,27 @@ export class MongoBucketBatch evaluated: paramEvaluated, existing_lookups }); - new_lookups = paramEvaluated.map((p) => { - return storage.serializeLookup(p.lookup); - }); + if (this.db.storageConfig.incrementalReprocessing) { + new_lookups = paramEvaluated.map((p) => { + const def = this.mapping.parameterLookupId(p.lookup.source); + return { d: def, l: storage.serializeLookup(p.lookup) } satisfies RecordedLookupV3; + }); + } else { + new_lookups = paramEvaluated.map((p) => { + return storage.serializeLookup(p.lookup); + }); + } } } - let result: CurrentDataDocument | null = null; + let result: CommonCurrentDataDocument | null = null; // 5. TOAST: Update current data and bucket list. if (afterId) { // Insert or update const after_key: SourceKey = { g: this.group_id, t: mongoTableId(sourceTable.id), k: afterId }; - batch.upsertCurrentData(after_key, { + batch.upsertCurrentData({ + id: after_key, data: afterData, buckets: new_buckets, lookups: new_lookups @@ -848,7 +886,7 @@ export class MongoBucketBatch await this.db.notifyCheckpoint(); this.persisted_op = null; this.last_checkpoint_lsn = lsn; - if (this.db.storageConfig.softDeleteCurrentData && newLastCheckpoint != null) { + if (this.db.storageConfig.incrementalReprocessing && newLastCheckpoint != null) { await this.cleanupCurrentData(newLastCheckpoint); } } @@ -1022,7 +1060,7 @@ export class MongoBucketBatch let lastBatchCount = BATCH_LIMIT; while (lastBatchCount == BATCH_LIMIT) { await this.withReplicationTransaction(`Truncate ${sourceTable.qualifiedName}`, async (session, opSeq) => { - const current_data_filter: mongo.Filter = { + const current_data_filter: mongo.Filter = { _id: idPrefixFilter({ g: this.group_id, t: mongoTableId(sourceTable.id) }, ['k']), // Skip soft-deleted data // Works for both v1 and v3 current_data schemas @@ -1039,7 +1077,7 @@ export class MongoBucketBatch session: session }); const batch = await cursor.toArray(); - const persistedBatch = new PersistedBatch(this.db, this.group_id, 0, { logger: this.logger }); + const persistedBatch = new PersistedBatch(this.db, this.group_id, this.mapping, 0, { logger: this.logger }); for (let value of batch) { persistedBatch.saveBucketData({ @@ -1205,6 +1243,7 @@ export class MongoBucketBatch } } -export function currentBucketKey(b: CurrentBucket) { - return `${b.bucket}/${b.table}/${b.id}`; +export function currentBucketKey(b: CommonCurrentBucket) { + const prefix = 'def' in b ? `${b.def}:` : ''; + return `${prefix}${b.bucket}/${b.table}/${b.id}`; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts new file mode 100644 index 000000000..ba5b4eb5e --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -0,0 +1,62 @@ +import { + BucketDataScope, + BucketDataSource, + CompatibilityOption, + DEFAULT_HYDRATION_STATE, + HydratedSyncRules, + HydrationState, + ParameterIndexLookupCreator, + SyncConfigWithErrors, + versionedHydrationState +} from '@powersync/service-sync-rules'; +import { storage } from '@powersync/service-core'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { StorageConfig } from './models.js'; + +export class MongoPersistedSyncRules implements storage.PersistedSyncRules { + public readonly hydrationState: HydrationState; + + constructor( + public readonly id: number, + public readonly sync_rules: SyncConfigWithErrors, + public readonly slot_name: string, + private readonly mapping: BucketDefinitionMapping | null, + private readonly storageConfig: StorageConfig + ) { + if (this.mapping != null && this.storageConfig.incrementalReprocessing) { + this.hydrationState = new MongoHydrationState(this.mapping); + } else if ( + !this.sync_rules.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && + !this.storageConfig.versionedBuckets + ) { + this.hydrationState = DEFAULT_HYDRATION_STATE; + } else { + this.hydrationState = versionedHydrationState(this.id); + } + } + + hydratedSyncRules(): HydratedSyncRules { + return this.sync_rules.config.hydrate({ hydrationState: this.hydrationState }); + } +} + +class MongoHydrationState implements HydrationState { + constructor(private readonly mapping: BucketDefinitionMapping) {} + + getBucketSourceScope(source: BucketDataSource): BucketDataScope { + const defId = this.mapping.bucketSourceId(source); + return { + bucketPrefix: defId.toString(16), + source + }; + } + + getParameterIndexLookupScope(source: ParameterIndexLookupCreator) { + const defId = this.mapping.parameterLookupId(source); + return { + lookupName: defId.toString(16), + queryId: '', + source + }; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts index 1595866af..e503a229d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts @@ -1,11 +1,14 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; import { MongoSyncRulesLock } from './MongoSyncRulesLock.js'; -import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js'; +import { PowerSyncMongo } from './db.js'; import { getMongoStorageConfig, SyncRuleDocument } from './models.js'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js'; export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesContent { public current_lock: MongoSyncRulesLock | null = null; + public readonly mapping: BucketDefinitionMapping; constructor( private db: PowerSyncMongo, @@ -25,12 +28,26 @@ export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesCo active: doc.state == 'ACTIVE', storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION }); + this.mapping = BucketDefinitionMapping.fromSyncRules(doc); } getStorageConfig() { return getMongoStorageConfig(this.storageVersion); } + parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules { + const parsed = super.parsed(options); + const storageConfig = this.getStorageConfig(); + + return new MongoPersistedSyncRules( + parsed.id, + parsed.sync_rules, + parsed.slot_name, + storageConfig.incrementalReprocessing ? this.mapping : null, + storageConfig + ); + } + async lock() { const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this); this.current_lock = lock; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index ce3e3f549..c6f4d9c0c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -36,8 +36,8 @@ import { BucketDataDocument, BucketDataKey, BucketStateDocument, + CommonSourceTableDocument, SourceKey, - SourceTableDocument, StorageConfig } from './models.js'; import { MongoBucketBatch } from './MongoBucketBatch.js'; @@ -180,6 +180,7 @@ export class MongoSyncBucketStorage logger: options.logger, db: this.db, syncRules: this.sync_rules.parsed(options).hydratedSyncRules(), + mapping: this.sync_rules.mapping, groupId: this.group_id, slotName: this.slot_name, lastCheckpointLsn: checkpoint_lsn, @@ -216,10 +217,11 @@ export class MongoSyncBucketStorage type: column.type, type_oid: column.typeId })); + const mapping = this.sync_rules.mapping; let result: storage.ResolveTableResult | null = null; await this.db.client.withSession(async (session) => { const col = this.db.source_tables; - let filter: Partial = { + let filter: Partial = { group_id: group_id, connection_id: connection_id, schema_name: schema, @@ -231,8 +233,17 @@ export class MongoSyncBucketStorage } let doc = await col.findOne(filter, { session }); if (doc == null) { - doc = { - _id: new bson.ObjectId(), + const candidateSourceTable = new storage.SourceTable({ + id: new bson.ObjectId(), + connectionTag: connection_tag, + objectId: objectId, + schema: schema, + name: name, + replicaIdColumns: replicaIdColumns, + snapshotComplete: false + }); + const createDoc: CommonSourceTableDocument = { + _id: candidateSourceTable.id as bson.ObjectId, group_id: group_id, connection_id: connection_id, relation_id: objectId, @@ -243,6 +254,20 @@ export class MongoSyncBucketStorage snapshot_done: false, snapshot_status: undefined }; + if (this.db.storageConfig.incrementalReprocessing) { + const bucketDataSourceIds = options.sync_rules.definition.bucketDataSources + .filter((source) => source.tableSyncsData(candidateSourceTable)) + .map((source) => mapping.bucketSourceId(source)); + const parameterLookupSourceIds = options.sync_rules.definition.bucketParameterLookupSources + .filter((source) => source.tableSyncsParameters(candidateSourceTable)) + .map((source) => mapping.parameterLookupId(source)); + + Object.assign(createDoc, { + bucket_data_source_ids: bucketDataSourceIds, + parameter_lookup_source_ids: parameterLookupSourceIds + }); + } + doc = createDoc; await col.insertOne(doc, { session }); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index ae7f75a2e..cde81abed 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -3,18 +3,24 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; +import { Logger, ReplicationAssertionError, logger as defaultLogger } from '@powersync/lib-services-framework'; import { InternalOpId, storage, utils } from '@powersync/service-core'; import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatch.js'; import { MongoIdSequence } from './MongoIdSequence.js'; -import { PowerSyncMongo, VersionedPowerSyncMongo } from './db.js'; +import { VersionedPowerSyncMongo } from './db.js'; import { BucketDataDocument, - BucketParameterDocument, BucketStateDocument, - CurrentBucket, + CommonBucketParameterDocument, + CommonCurrentBucket, + CommonCurrentLookup, + CommonCurrentDataDocument, CurrentDataDocument, - SourceKey + CurrentDataDocumentV3, + RecordedLookupV3, + SourceKey, + isCurrentBucketV3, + isRecordedLookupV3 } from './models.js'; import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; @@ -48,8 +54,8 @@ const MAX_TRANSACTION_DOC_COUNT = 2_000; export class PersistedBatch { logger: Logger; bucketData: mongo.AnyBulkWriteOperation[] = []; - bucketParameters: mongo.AnyBulkWriteOperation[] = []; - currentData: mongo.AnyBulkWriteOperation[] = []; + bucketParameters: mongo.AnyBulkWriteOperation[] = []; + currentData: mongo.AnyBulkWriteOperation[] = []; bucketStates: Map = new Map(); /** @@ -92,9 +98,9 @@ export class PersistedBatch { sourceKey: storage.ReplicaId; table: storage.SourceTable; evaluated: EvaluatedRow[]; - before_buckets: CurrentBucket[]; + before_buckets: CommonCurrentBucket[]; }) { - const remaining_buckets = new Map(); + const remaining_buckets = new Map(); for (let b of options.before_buckets) { const key = currentBucketKey(b); remaining_buckets.set(key, b); @@ -103,7 +109,20 @@ export class PersistedBatch { const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); for (const k of options.evaluated) { - const key = currentBucketKey(k); + const key = currentBucketKey( + this.db.storageConfig.incrementalReprocessing + ? { + def: getDefinitionIdFromBucketName(k.bucket), + bucket: k.bucket, + table: k.table, + id: k.id + } + : { + bucket: k.bucket, + table: k.table, + id: k.id + } + ); // INSERT const recordData = JSONBig.stringify(k.data); @@ -179,7 +198,7 @@ export class PersistedBatch { sourceKey: storage.ReplicaId; sourceTable: storage.SourceTable; evaluated: EvaluatedParameters[]; - existing_lookups: bson.Binary[]; + existing_lookups: CommonCurrentLookup[]; }) { // This is similar to saving bucket data. // A key difference is that we don't need to keep the history intact. @@ -190,31 +209,59 @@ export class PersistedBatch { // We also don't need to keep history intact. const { sourceTable, sourceKey, evaluated } = data; - const remaining_lookups = new Map(); + const remaining_lookups = new Map(); for (let l of data.existing_lookups) { - remaining_lookups.set(l.toString('base64'), l); + if (this.db.storageConfig.incrementalReprocessing) { + if (!isRecordedLookupV3(l)) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + remaining_lookups.set(`${l.d}.${l.l.toString('base64')}`, l); + } else { + remaining_lookups.set(l.toString('base64'), l); + } } // 1. Insert new entries for (let result of evaluated) { const binLookup = storage.serializeLookup(result.lookup); - const hex = binLookup.toString('base64'); - remaining_lookups.delete(hex); + let sourceDefinitionId: number | undefined = undefined; + if (this.db.storageConfig.incrementalReprocessing) { + sourceDefinitionId = getDefinitionIdFromLookup(result.lookup); + remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`); + } else { + remaining_lookups.delete(binLookup.toString('base64')); + } const op_id = data.op_seq.next(); this.debugLastOpId = op_id; + const key = { + g: this.group_id, + t: mongoTableId(sourceTable.id), + k: sourceKey + }; + let values: CommonBucketParameterDocument; + if (this.db.storageConfig.incrementalReprocessing) { + if (sourceDefinitionId == null) { + throw new ReplicationAssertionError('Missing parameter lookup source mapping'); + } + values = { + _id: op_id, + def: sourceDefinitionId, + key, + lookup: binLookup, + bucket_parameters: result.bucketParameters + }; + } else { + values = { + _id: op_id, + key, + lookup: binLookup, + bucket_parameters: result.bucketParameters + }; + } this.bucketParameters.push({ insertOne: { - document: { - _id: op_id, - key: { - g: this.group_id, - t: mongoTableId(sourceTable.id), - k: sourceKey - }, - lookup: binLookup, - bucket_parameters: result.bucketParameters - } + document: values } }); @@ -223,20 +270,39 @@ export class PersistedBatch { // 2. "REMOVE" entries for any lookup not touched. for (let lookup of remaining_lookups.values()) { + const key = { + g: this.group_id, + t: mongoTableId(sourceTable.id), + k: sourceKey + }; const op_id = data.op_seq.next(); this.debugLastOpId = op_id; + let values: CommonBucketParameterDocument; + if (this.db.storageConfig.incrementalReprocessing) { + if (!isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + values = { + _id: op_id, + def: lookup.d, + key, + lookup: lookup.l, + bucket_parameters: [] + }; + } else { + if (isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); + } + values = { + _id: op_id, + key, + lookup, + bucket_parameters: [] + }; + } this.bucketParameters.push({ insertOne: { - document: { - _id: op_id, - key: { - g: this.group_id, - t: mongoTableId(sourceTable.id), - k: sourceKey - }, - lookup: lookup, - bucket_parameters: [] - } + document: values } }); @@ -245,7 +311,7 @@ export class PersistedBatch { } hardDeleteCurrentData(id: SourceKey) { - const op: mongo.AnyBulkWriteOperation = { + const op: mongo.AnyBulkWriteOperation = { deleteOne: { filter: { _id: id } } @@ -257,21 +323,21 @@ export class PersistedBatch { /** * Mark a current_data document as soft deleted, to delete on the next commit. * - * If softDeleteCurrentData is not enabled, this falls back to a hard delete. + * If incremental reprocessing is not enabled, this falls back to a hard delete. */ softDeleteCurrentData(id: SourceKey, checkpointGreaterThan: bigint) { - if (!this.db.storageConfig.softDeleteCurrentData) { + if (!this.db.storageConfig.incrementalReprocessing) { this.hardDeleteCurrentData(id); return; } - const op: mongo.AnyBulkWriteOperation = { + const op: mongo.AnyBulkWriteOperation = { updateOne: { filter: { _id: id }, update: { $set: { data: EMPTY_DATA, - buckets: [], - lookups: [], + buckets: [] as CurrentDataDocumentV3['buckets'], + lookups: [] as CurrentDataDocumentV3['lookups'], pending_delete: checkpointGreaterThan } }, @@ -282,18 +348,68 @@ export class PersistedBatch { this.currentSize += 50; } - upsertCurrentData(id: SourceKey, values: Partial) { - const op: mongo.AnyBulkWriteOperation = { - updateOne: { - filter: { _id: id }, - update: { - $set: values, - $unset: { pending_delete: 1 } - }, - upsert: true - } - }; - this.currentData.push(op); + upsertCurrentData(values: { + id: SourceKey; + data: bson.Binary | undefined; + buckets: CommonCurrentBucket[]; + lookups: CommonCurrentLookup[]; + }) { + if (this.db.storageConfig.incrementalReprocessing) { + const buckets = values.buckets.map((bucket) => { + if (!isCurrentBucketV3(bucket)) { + throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); + } + return bucket; + }); + const lookups = values.lookups.map((lookup) => { + if (!isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + return lookup; + }); + const op: mongo.AnyBulkWriteOperation = { + updateOne: { + filter: { _id: values.id }, + update: { + $set: { + data: values.data, + buckets, + lookups + }, + $unset: { pending_delete: 1 } + }, + upsert: true + } + }; + this.currentData.push(op); + } else { + const buckets = values.buckets.map((bucket) => ({ + bucket: bucket.bucket, + table: bucket.table, + id: bucket.id + })); + const lookups = values.lookups.map((lookup) => { + if (isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); + } + return lookup; + }); + const op: mongo.AnyBulkWriteOperation = { + updateOne: { + filter: { _id: values.id }, + update: { + $set: { + data: values.data, + buckets, + lookups + }, + $unset: { pending_delete: 1 } + }, + upsert: true + } + }; + this.currentData.push(op); + } this.currentSize += (values.data?.length() ?? 0) + 100; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index d3b2e8b02..5bc0f8281 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -6,15 +6,20 @@ import { MongoStorageConfig } from '../../types/types.js'; import { BucketDataDocument, BucketParameterDocument, + BucketParameterDocumentV3, BucketStateDocument, CheckpointEventDocument, ClientConnectionDocument, + CommonBucketParameterDocument, + CommonCurrentDataDocument, + CommonSourceTableDocument, CurrentDataDocument, CurrentDataDocumentV3, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, SourceTableDocument, + SourceTableDocumentV3, StorageConfig, SyncRuleDocument, WriteCheckpointDocument @@ -33,9 +38,11 @@ export class PowerSyncMongo { readonly v3_current_data: mongo.Collection; readonly bucket_data: mongo.Collection; readonly bucket_parameters: mongo.Collection; + readonly v3_bucket_parameters: mongo.Collection; readonly op_id_sequence: mongo.Collection; readonly sync_rules: mongo.Collection; readonly source_tables: mongo.Collection; + readonly v3_source_tables: mongo.Collection; readonly custom_write_checkpoints: mongo.Collection; readonly write_checkpoints: mongo.Collection; readonly instance: mongo.Collection; @@ -59,9 +66,11 @@ export class PowerSyncMongo { this.v3_current_data = db.collection('v3_current_data'); this.bucket_data = db.collection('bucket_data'); this.bucket_parameters = db.collection('bucket_parameters'); + this.v3_bucket_parameters = db.collection('v3_bucket_parameters'); this.op_id_sequence = db.collection('op_id_sequence'); this.sync_rules = db.collection('sync_rules'); this.source_tables = db.collection('source_tables'); + this.v3_source_tables = db.collection('v3_source_tables'); this.custom_write_checkpoints = db.collection('custom_write_checkpoints'); this.write_checkpoints = db.collection('write_checkpoints'); this.instance = db.collection('instance'); @@ -83,9 +92,11 @@ export class PowerSyncMongo { await this.v3_current_data.deleteMany({}); await this.bucket_data.deleteMany({}); await this.bucket_parameters.deleteMany({}); + await this.v3_bucket_parameters.deleteMany({}); await this.op_id_sequence.deleteMany({}); await this.sync_rules.deleteMany({}); await this.source_tables.deleteMany({}); + await this.v3_source_tables.deleteMany({}); await this.write_checkpoints.deleteMany({}); await this.instance.deleteOne({}); await this.locks.deleteMany({}); @@ -183,7 +194,7 @@ export class PowerSyncMongo { } async initializeStorageVersion(storageConfig: StorageConfig) { - if (storageConfig.softDeleteCurrentData) { + if (storageConfig.incrementalReprocessing) { // Initialize the v3_current_data collection, which is used for the new storage version. // No-op if this already exists await this.v3_current_data.createIndex( @@ -196,6 +207,28 @@ export class PowerSyncMongo { name: 'pending_delete' } ); + await this.v3_bucket_parameters.createIndex( + { + 'key.g': 1, + lookup: 1, + _id: 1 + }, + { + name: 'lookup_group_id' + } + ); + await this.v3_source_tables.createIndex( + { + group_id: 1, + connection_id: 1, + schema_name: 1, + table_name: 1, + relation_id: 1 + }, + { + name: 'source_lookup' + } + ); } } } @@ -222,27 +255,27 @@ export class VersionedPowerSyncMongo { * * Use in places where it does not matter which version is used. */ - get common_current_data(): mongo.Collection { - if (this.storageConfig.softDeleteCurrentData) { - return this.#upstream.v3_current_data; + get common_current_data(): mongo.Collection { + if (this.storageConfig.incrementalReprocessing) { + return this.#upstream.v3_current_data as unknown as mongo.Collection; } else { - return this.#upstream.current_data; + return this.#upstream.current_data as unknown as mongo.Collection; } } get v1_current_data() { - if (this.storageConfig.softDeleteCurrentData) { + if (this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( - 'current_data collection should not be used when softDeleteCurrentData is enabled' + 'current_data collection should not be used when incrementalReprocessing is enabled' ); } return this.#upstream.current_data; } get v3_current_data() { - if (!this.storageConfig.softDeleteCurrentData) { + if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( - 'v3_current_data collection should not be used when softDeleteCurrentData is disabled' + 'v3_current_data collection should not be used when incrementalReprocessing is disabled' ); } return this.#upstream.v3_current_data; @@ -253,7 +286,11 @@ export class VersionedPowerSyncMongo { } get bucket_parameters() { - return this.#upstream.bucket_parameters; + if (this.storageConfig.incrementalReprocessing) { + return this.#upstream.v3_bucket_parameters as unknown as mongo.Collection; + } else { + return this.#upstream.bucket_parameters as unknown as mongo.Collection; + } } get op_id_sequence() { @@ -265,7 +302,11 @@ export class VersionedPowerSyncMongo { } get source_tables() { - return this.#upstream.source_tables; + if (this.storageConfig.incrementalReprocessing) { + return this.#upstream.v3_source_tables as unknown as mongo.Collection; + } else { + return this.#upstream.source_tables as unknown as mongo.Collection; + } } get custom_write_checkpoints() { diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 90382ad80..c020c0a8b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -38,11 +38,20 @@ export interface CurrentDataDocument { lookups: bson.Binary[]; } +export interface CurrentBucketV3 extends CurrentBucket { + def: number; +} + +export interface RecordedLookupV3 { + d: number; + l: bson.Binary; +} + export interface CurrentDataDocumentV3 { _id: SourceKey; data: bson.Binary; - buckets: CurrentBucket[]; - lookups: bson.Binary[]; + buckets: CurrentBucketV3[]; + lookups: RecordedLookupV3[]; /** * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete. * @@ -64,6 +73,10 @@ export interface BucketParameterDocument { bucket_parameters: Record[]; } +export interface BucketParameterDocumentV3 extends BucketParameterDocument { + def: number; +} + export interface BucketDataDocument { _id: BucketDataKey; op: OpType; @@ -91,6 +104,11 @@ export interface SourceTableDocument { snapshot_status: SourceTableDocumentSnapshotStatus | undefined; } +export interface SourceTableDocumentV3 extends SourceTableDocument { + bucket_data_source_ids: number[]; + parameter_lookup_source_ids: number[]; +} + export interface SourceTableDocumentSnapshotStatus { total_estimated_count: number; replicated_count: number; @@ -214,6 +232,10 @@ export interface SyncRuleDocument { content: string; serialized_plan?: SerializedSyncPlan | null; + rule_mapping?: { + definitions: Record; + parameter_lookups: Record; + }; lock?: { id: string; @@ -231,9 +253,14 @@ export interface StorageConfig extends storage.StorageVersionConfig { * a Long before summing. */ longChecksums: boolean; + /** + * Enables v3 MongoDB storage behavior used for incremental reprocessing. + */ + incrementalReprocessing: boolean; } const LONG_CHECKSUMS_STORAGE_VERSION = 2; +const INCREMENTAL_REPROCESSING_STORAGE_VERSION = storage.STORAGE_VERSION_3; export function getMongoStorageConfig(storageVersion: number): StorageConfig { const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion]; @@ -241,7 +268,11 @@ export function getMongoStorageConfig(storageVersion: number): StorageConfig { throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${storageVersion}`); } - return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION }; + return { + ...baseConfig, + longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION, + incrementalReprocessing: storageVersion >= INCREMENTAL_REPROCESSING_STORAGE_VERSION + }; } export interface CheckpointEventDocument { @@ -286,3 +317,17 @@ export interface InstanceDocument { } export interface ClientConnectionDocument extends event_types.ClientConnection {} + +export type CommonCurrentDataDocument = CurrentDataDocument | CurrentDataDocumentV3; +export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3; +export type CommonCurrentLookup = bson.Binary | RecordedLookupV3; +export type CommonBucketParameterDocument = BucketParameterDocument | BucketParameterDocumentV3; +export type CommonSourceTableDocument = SourceTableDocument | SourceTableDocumentV3; + +export function isCurrentBucketV3(bucket: CommonCurrentBucket): bucket is CurrentBucketV3 { + return 'def' in bucket; +} + +export function isRecordedLookupV3(lookup: CommonCurrentLookup): lookup is RecordedLookupV3 { + return !(lookup instanceof bson.Binary); +} diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index 1c754cb5f..4ba34a0cf 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -1,7 +1,9 @@ export * from './implementation/db.js'; +export * from './implementation/BucketDefinitionMapping.js'; export * from './implementation/models.js'; export * from './implementation/MongoBucketBatch.js'; export * from './implementation/MongoIdSequence.js'; +export * from './implementation/MongoPersistedSyncRules.js'; export * from './implementation/MongoPersistedSyncRulesContent.js'; export * from './implementation/MongoStorageProvider.js'; export * from './implementation/MongoSyncBucketStorage.js'; diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 7e53340c8..e0f032808 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -2,6 +2,8 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; +import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; +import { CurrentDataDocumentV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) { register.registerSyncTests(storageConfig.factory, { @@ -126,12 +128,47 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor // Test that the checksum type is correct. // Specifically, test that it never persisted as double. - const mongoFactory = factory as any; + const mongoFactory = factory as MongoBucketStorage; const checksumTypes = await mongoFactory.db.bucket_data .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }]) .toArray(); expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]); }); + + test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM "%" + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules); + + await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { + await batch.save({ + sourceTable: TEST_TABLE, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'shape-check', + description: 'shape' + }, + afterReplicaId: test_utils.rid('shape-check') + }); + }); + + const mongoFactory = factory as MongoBucketStorage; + const currentData = (await mongoFactory.db.v3_current_data.findOne({})) as CurrentDataDocumentV3 | null; + expect(currentData?.buckets?.[0]?.def).toBeGreaterThan(0); + + const syncRule = (await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id })) as SyncRuleDocument | null; + expect(Object.keys(syncRule?.rule_mapping?.definitions ?? {})).not.toHaveLength(0); + }); } describe('sync - mongodb', () => { From d3e3d31c3cdbdc7486b1f6e52a9d3f8e3621684b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 5 Mar 2026 16:10:25 +0200 Subject: [PATCH 02/93] Handle current_data v1/v3 differences. --- .../implementation/MongoBucketBatch.ts | 45 +++++++++-- .../storage/implementation/PersistedBatch.ts | 78 ++++++++++++------- 2 files changed, 90 insertions(+), 33 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 4afaf8f98..529df9dbf 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -592,12 +592,45 @@ export class MongoBucketBatch buckets: new_buckets, lookups: new_lookups }); - result = { - _id: after_key, - data: afterData!, - buckets: new_buckets, - lookups: new_lookups - }; + if (this.db.storageConfig.incrementalReprocessing) { + const buckets = new_buckets.map((bucket) => { + if (!('def' in bucket)) { + throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); + } + return bucket; + }); + const lookups = new_lookups.map((lookup) => { + if (lookup instanceof bson.Binary) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + return lookup; + }); + result = { + _id: after_key, + data: afterData!, + buckets, + lookups + }; + } else { + const buckets = new_buckets.map((bucket) => { + if ('def' in bucket) { + throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); + } + return bucket; + }); + const lookups = new_lookups.map((lookup) => { + if (!(lookup instanceof bson.Binary)) { + throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); + } + return lookup; + }); + result = { + _id: after_key, + data: afterData!, + buckets, + lookups + }; + } } if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index cde81abed..0a22030e4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -8,13 +8,13 @@ import { InternalOpId, storage, utils } from '@powersync/service-core'; import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatch.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { VersionedPowerSyncMongo } from './db.js'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { BucketDataDocument, BucketStateDocument, CommonBucketParameterDocument, CommonCurrentBucket, CommonCurrentLookup, - CommonCurrentDataDocument, CurrentDataDocument, CurrentDataDocumentV3, RecordedLookupV3, @@ -55,7 +55,8 @@ export class PersistedBatch { logger: Logger; bucketData: mongo.AnyBulkWriteOperation[] = []; bucketParameters: mongo.AnyBulkWriteOperation[] = []; - currentData: mongo.AnyBulkWriteOperation[] = []; + currentDataV1: mongo.AnyBulkWriteOperation[] = []; + currentDataV3: mongo.AnyBulkWriteOperation[] = []; bucketStates: Map = new Map(); /** @@ -71,6 +72,7 @@ export class PersistedBatch { constructor( private db: VersionedPowerSyncMongo, private group_id: number, + private mapping: BucketDefinitionMapping, writtenSize: number, options?: { logger?: Logger } ) { @@ -109,10 +111,11 @@ export class PersistedBatch { const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); for (const k of options.evaluated) { + const sourceDefinitionId = this.mapping.bucketSourceId(k.source); const key = currentBucketKey( this.db.storageConfig.incrementalReprocessing ? { - def: getDefinitionIdFromBucketName(k.bucket), + def: sourceDefinitionId, bucket: k.bucket, table: k.table, id: k.id @@ -226,7 +229,7 @@ export class PersistedBatch { const binLookup = storage.serializeLookup(result.lookup); let sourceDefinitionId: number | undefined = undefined; if (this.db.storageConfig.incrementalReprocessing) { - sourceDefinitionId = getDefinitionIdFromLookup(result.lookup); + sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source); remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`); } else { remaining_lookups.delete(binLookup.toString('base64')); @@ -311,12 +314,21 @@ export class PersistedBatch { } hardDeleteCurrentData(id: SourceKey) { - const op: mongo.AnyBulkWriteOperation = { - deleteOne: { - filter: { _id: id } - } - }; - this.currentData.push(op); + if (this.db.storageConfig.incrementalReprocessing) { + const op: mongo.AnyBulkWriteOperation = { + deleteOne: { + filter: { _id: id } + } + }; + this.currentDataV3.push(op); + } else { + const op: mongo.AnyBulkWriteOperation = { + deleteOne: { + filter: { _id: id } + } + }; + this.currentDataV1.push(op); + } this.currentSize += 50; } @@ -344,7 +356,7 @@ export class PersistedBatch { upsert: true } }; - this.currentData.push(op); + this.currentDataV3.push(op); this.currentSize += 50; } @@ -381,7 +393,7 @@ export class PersistedBatch { upsert: true } }; - this.currentData.push(op); + this.currentDataV3.push(op); } else { const buckets = values.buckets.map((bucket) => ({ bucket: bucket.bucket, @@ -408,7 +420,7 @@ export class PersistedBatch { upsert: true } }; - this.currentData.push(op); + this.currentDataV1.push(op); } this.currentSize += (values.data?.length() ?? 0) + 100; } @@ -417,7 +429,7 @@ export class PersistedBatch { return ( this.currentSize >= MAX_TRANSACTION_BATCH_SIZE || this.bucketData.length >= MAX_TRANSACTION_DOC_COUNT || - this.currentData.length >= MAX_TRANSACTION_DOC_COUNT || + this.currentDataV1.length + this.currentDataV3.length >= MAX_TRANSACTION_DOC_COUNT || this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT ); } @@ -442,13 +454,24 @@ export class PersistedBatch { ordered: false }); } - if (this.currentData.length > 0) { - flushedSomething = true; - await db.common_current_data.bulkWrite(this.currentData, { - session, - // may update and delete data within the same batch - order matters - ordered: true - }); + if (this.db.storageConfig.incrementalReprocessing) { + if (this.currentDataV3.length > 0) { + flushedSomething = true; + await db.v3_current_data.bulkWrite(this.currentDataV3, { + session, + // may update and delete data within the same batch - order matters + ordered: true + }); + } + } else { + if (this.currentDataV1.length > 0) { + flushedSomething = true; + await db.v1_current_data.bulkWrite(this.currentDataV1, { + session, + // may update and delete data within the same batch - order matters + ordered: true + }); + } } if (this.bucketStates.size > 0) { @@ -467,7 +490,7 @@ export class PersistedBatch { this.logger.info( `Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${ - this.currentData.length + this.currentDataV1.length + this.currentDataV3.length } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`, { flushed: { @@ -475,7 +498,7 @@ export class PersistedBatch { size: this.currentSize, bucket_data_count: this.bucketData.length, parameter_data_count: this.bucketParameters.length, - current_data_count: this.currentData.length, + current_data_count: this.currentDataV1.length + this.currentDataV3.length, replication_lag_seconds: replicationLag } } @@ -483,7 +506,7 @@ export class PersistedBatch { } else { this.logger.info( `Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${ - this.currentData.length + this.currentDataV1.length + this.currentDataV3.length } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`, { flushed: { @@ -491,7 +514,7 @@ export class PersistedBatch { size: this.currentSize, bucket_data_count: this.bucketData.length, parameter_data_count: this.bucketParameters.length, - current_data_count: this.currentData.length + current_data_count: this.currentDataV1.length + this.currentDataV3.length } } ); @@ -501,13 +524,14 @@ export class PersistedBatch { const stats = { bucketDataCount: this.bucketData.length, parameterDataCount: this.bucketParameters.length, - currentDataCount: this.currentData.length, + currentDataCount: this.currentDataV1.length + this.currentDataV3.length, flushedAny: flushedSomething }; this.bucketData = []; this.bucketParameters = []; - this.currentData = []; + this.currentDataV1 = []; + this.currentDataV3 = []; this.bucketStates.clear(); this.currentSize = 0; this.debugLastOpId = null; From 20a99b2b7060f3e1e364f922ea169e790efdd1ab Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 12:01:51 +0200 Subject: [PATCH 03/93] Split out PersistedBatch implementations. --- .../implementation/MongoBucketBatch.ts | 25 +- .../storage/implementation/PersistedBatch.ts | 480 +++++------------- .../implementation/PersistedBatchV1.ts | 207 ++++++++ .../implementation/PersistedBatchV3.ts | 225 ++++++++ 4 files changed, 561 insertions(+), 376 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 529df9dbf..d664f59fd 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -39,6 +39,8 @@ import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; +import { PersistedBatchV1 } from './PersistedBatchV1.js'; +import { PersistedBatchV3 } from './PersistedBatchV3.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; /** @@ -159,6 +161,17 @@ export class MongoBucketBatch return this.last_checkpoint_lsn; } + private createPersistedBatch(writtenSize: number): PersistedBatch { + if (this.db.storageConfig.incrementalReprocessing) { + return new PersistedBatchV3(this.db, this.group_id, this.mapping, writtenSize, { + logger: this.logger + }); + } + return new PersistedBatchV1(this.db, this.group_id, this.mapping, writtenSize, { + logger: this.logger + }); + } + async flush(options?: storage.BatchBucketFlushOptions): Promise { let result: storage.FlushedResult | null = null; // One flush may be split over multiple transactions. @@ -284,15 +297,7 @@ export class MongoBucketBatch current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc); } - let persistedBatch: PersistedBatch | null = new PersistedBatch( - this.db, - this.group_id, - this.mapping, - transactionSize, - { - logger: this.logger - } - ); + let persistedBatch: PersistedBatch | null = this.createPersistedBatch(transactionSize); for (let op of b) { if (resumeBatch) { @@ -1110,7 +1115,7 @@ export class MongoBucketBatch session: session }); const batch = await cursor.toArray(); - const persistedBatch = new PersistedBatch(this.db, this.group_id, this.mapping, 0, { logger: this.logger }); + const persistedBatch = this.createPersistedBatch(0); for (let value of batch) { persistedBatch.saveBucketData({ diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index 0a22030e4..3a96db749 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -1,11 +1,9 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { JSONBig } from '@powersync/service-jsonbig'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { Logger, ReplicationAssertionError, logger as defaultLogger } from '@powersync/lib-services-framework'; -import { InternalOpId, storage, utils } from '@powersync/service-core'; -import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatch.js'; +import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; +import { InternalOpId, storage } from '@powersync/service-core'; import { MongoIdSequence } from './MongoIdSequence.js'; import { VersionedPowerSyncMongo } from './db.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; @@ -15,14 +13,9 @@ import { CommonBucketParameterDocument, CommonCurrentBucket, CommonCurrentLookup, - CurrentDataDocument, - CurrentDataDocumentV3, - RecordedLookupV3, - SourceKey, - isCurrentBucketV3, - isRecordedLookupV3 + SourceKey } from './models.js'; -import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; +import { mongoTableId } from '../../utils/util.js'; /** * Maximum size of operations we write in a single transaction. @@ -45,18 +38,43 @@ const MAX_TRANSACTION_BATCH_SIZE = 30_000_000; */ const MAX_TRANSACTION_DOC_COUNT = 2_000; +export interface SaveBucketDataOptions { + op_seq: MongoIdSequence; + sourceKey: storage.ReplicaId; + table: storage.SourceTable; + evaluated: EvaluatedRow[]; + before_buckets: CommonCurrentBucket[]; +} + +export interface SaveParameterDataOptions { + op_seq: MongoIdSequence; + sourceKey: storage.ReplicaId; + sourceTable: storage.SourceTable; + evaluated: EvaluatedParameters[]; + existing_lookups: CommonCurrentLookup[]; +} + +export interface UpsertCurrentDataOptions { + id: SourceKey; + data: bson.Binary | undefined; + buckets: CommonCurrentBucket[]; + lookups: CommonCurrentLookup[]; +} + +export interface PersistedBatchOptions { + logger?: Logger; +} + /** * Keeps track of bulkwrite operations within a transaction. * * There may be multiple of these batches per transaction, but it may not span * multiple transactions. */ -export class PersistedBatch { +export abstract class PersistedBatch { logger: Logger; bucketData: mongo.AnyBulkWriteOperation[] = []; bucketParameters: mongo.AnyBulkWriteOperation[] = []; - currentDataV1: mongo.AnyBulkWriteOperation[] = []; - currentDataV3: mongo.AnyBulkWriteOperation[] = []; bucketStates: Map = new Map(); /** @@ -70,17 +88,33 @@ export class PersistedBatch { currentSize = 0; constructor( - private db: VersionedPowerSyncMongo, - private group_id: number, - private mapping: BucketDefinitionMapping, + protected readonly db: VersionedPowerSyncMongo, + protected readonly group_id: number, + protected readonly mapping: BucketDefinitionMapping, writtenSize: number, - options?: { logger?: Logger } + options?: PersistedBatchOptions ) { this.currentSize = writtenSize; this.logger = options?.logger ?? defaultLogger; } - private incrementBucket(bucket: string, op_id: InternalOpId, bytes: number) { + abstract saveBucketData(options: SaveBucketDataOptions): void; + + abstract saveParameterData(data: SaveParameterDataOptions): void; + + abstract hardDeleteCurrentData(id: SourceKey): void; + + abstract softDeleteCurrentData(id: SourceKey, checkpointGreaterThan: bigint): void; + + abstract upsertCurrentData(values: UpsertCurrentDataOptions): void; + + protected abstract get currentDataCount(): number; + + protected abstract flushCurrentData(session: mongo.ClientSession): Promise; + + protected abstract resetCurrentData(): void; + + protected incrementBucket(bucket: string, op_id: InternalOpId, bytes: number) { let existingState = this.bucketStates.get(bucket); if (existingState) { existingState.lastOp = op_id; @@ -95,341 +129,74 @@ export class PersistedBatch { } } - saveBucketData(options: { - op_seq: MongoIdSequence; + protected addBucketDataPut(options: { + op_id: InternalOpId; + bucket: string; + sourceTableId: storage.SourceTable['id']; sourceKey: storage.ReplicaId; - table: storage.SourceTable; - evaluated: EvaluatedRow[]; - before_buckets: CommonCurrentBucket[]; + table: string; + rowId: string; + checksum: bigint; + data: string; }) { - const remaining_buckets = new Map(); - for (let b of options.before_buckets) { - const key = currentBucketKey(b); - remaining_buckets.set(key, b); - } - - const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); - - for (const k of options.evaluated) { - const sourceDefinitionId = this.mapping.bucketSourceId(k.source); - const key = currentBucketKey( - this.db.storageConfig.incrementalReprocessing - ? { - def: sourceDefinitionId, - bucket: k.bucket, - table: k.table, - id: k.id - } - : { - bucket: k.bucket, - table: k.table, - id: k.id - } - ); - - // INSERT - const recordData = JSONBig.stringify(k.data); - const checksum = utils.hashData(k.table, k.id, recordData); - if (recordData.length > MAX_ROW_SIZE) { - // In many cases, the raw data size would have been too large already. But there are cases where - // the BSON size is small enough, but the JSON size is too large. - // In these cases, we can't store the data, so we skip it, or generate a REMOVE operation if the row - // was synced previously. - this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`); - continue; - } - - remaining_buckets.delete(key); - const byteEstimate = recordData.length + 200; - this.currentSize += byteEstimate; - - const op_id = options.op_seq.next(); - this.debugLastOpId = op_id; - - this.bucketData.push({ - insertOne: { - document: { - _id: { - g: this.group_id, - b: k.bucket, - o: op_id - }, - op: 'PUT', - source_table: mongoTableId(options.table.id), - source_key: options.sourceKey, - table: k.table, - row_id: k.id, - checksum: BigInt(checksum), - data: recordData - } - } - }); - this.incrementBucket(k.bucket, op_id, byteEstimate); - } - - for (let bd of remaining_buckets.values()) { - // REMOVE - - const op_id = options.op_seq.next(); - this.debugLastOpId = op_id; - - this.bucketData.push({ - insertOne: { - document: { - _id: { - g: this.group_id, - b: bd.bucket, - o: op_id - }, - op: 'REMOVE', - source_table: mongoTableId(options.table.id), - source_key: options.sourceKey, - table: bd.table, - row_id: bd.id, - checksum: dchecksum, - data: null - } + this.bucketData.push({ + insertOne: { + document: { + _id: { + g: this.group_id, + b: options.bucket, + o: options.op_id + }, + op: 'PUT', + source_table: mongoTableId(options.sourceTableId), + source_key: options.sourceKey, + table: options.table, + row_id: options.rowId, + checksum: options.checksum, + data: options.data } - }); - this.currentSize += 200; - this.incrementBucket(bd.bucket, op_id, 200); - } + } + }); } - saveParameterData(data: { - op_seq: MongoIdSequence; + protected addBucketDataRemove(options: { + op_id: InternalOpId; + bucket: string; + sourceTableId: storage.SourceTable['id']; sourceKey: storage.ReplicaId; - sourceTable: storage.SourceTable; - evaluated: EvaluatedParameters[]; - existing_lookups: CommonCurrentLookup[]; + table: string; + rowId: string; + checksum: bigint; }) { - // This is similar to saving bucket data. - // A key difference is that we don't need to keep the history intact. - // We do need to keep track of recent history though - enough that we can get consistent data for any specific checkpoint. - // Instead of storing per bucket id, we store per "lookup". - // A key difference is that we don't need to store or keep track of anything per-bucket - the entire record is - // either persisted or removed. - // We also don't need to keep history intact. - const { sourceTable, sourceKey, evaluated } = data; - - const remaining_lookups = new Map(); - for (let l of data.existing_lookups) { - if (this.db.storageConfig.incrementalReprocessing) { - if (!isRecordedLookupV3(l)) { - throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); - } - remaining_lookups.set(`${l.d}.${l.l.toString('base64')}`, l); - } else { - remaining_lookups.set(l.toString('base64'), l); - } - } - - // 1. Insert new entries - for (let result of evaluated) { - const binLookup = storage.serializeLookup(result.lookup); - let sourceDefinitionId: number | undefined = undefined; - if (this.db.storageConfig.incrementalReprocessing) { - sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source); - remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`); - } else { - remaining_lookups.delete(binLookup.toString('base64')); - } - - const op_id = data.op_seq.next(); - this.debugLastOpId = op_id; - const key = { - g: this.group_id, - t: mongoTableId(sourceTable.id), - k: sourceKey - }; - let values: CommonBucketParameterDocument; - if (this.db.storageConfig.incrementalReprocessing) { - if (sourceDefinitionId == null) { - throw new ReplicationAssertionError('Missing parameter lookup source mapping'); - } - values = { - _id: op_id, - def: sourceDefinitionId, - key, - lookup: binLookup, - bucket_parameters: result.bucketParameters - }; - } else { - values = { - _id: op_id, - key, - lookup: binLookup, - bucket_parameters: result.bucketParameters - }; - } - this.bucketParameters.push({ - insertOne: { - document: values - } - }); - - this.currentSize += 200; - } - - // 2. "REMOVE" entries for any lookup not touched. - for (let lookup of remaining_lookups.values()) { - const key = { - g: this.group_id, - t: mongoTableId(sourceTable.id), - k: sourceKey - }; - const op_id = data.op_seq.next(); - this.debugLastOpId = op_id; - let values: CommonBucketParameterDocument; - if (this.db.storageConfig.incrementalReprocessing) { - if (!isRecordedLookupV3(lookup)) { - throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); - } - values = { - _id: op_id, - def: lookup.d, - key, - lookup: lookup.l, - bucket_parameters: [] - }; - } else { - if (isRecordedLookupV3(lookup)) { - throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); - } - values = { - _id: op_id, - key, - lookup, - bucket_parameters: [] - }; - } - this.bucketParameters.push({ - insertOne: { - document: values - } - }); - - this.currentSize += 200; - } - } - - hardDeleteCurrentData(id: SourceKey) { - if (this.db.storageConfig.incrementalReprocessing) { - const op: mongo.AnyBulkWriteOperation = { - deleteOne: { - filter: { _id: id } - } - }; - this.currentDataV3.push(op); - } else { - const op: mongo.AnyBulkWriteOperation = { - deleteOne: { - filter: { _id: id } + this.bucketData.push({ + insertOne: { + document: { + _id: { + g: this.group_id, + b: options.bucket, + o: options.op_id + }, + op: 'REMOVE', + source_table: mongoTableId(options.sourceTableId), + source_key: options.sourceKey, + table: options.table, + row_id: options.rowId, + checksum: options.checksum, + data: null } - }; - this.currentDataV1.push(op); - } - this.currentSize += 50; - } - - /** - * Mark a current_data document as soft deleted, to delete on the next commit. - * - * If incremental reprocessing is not enabled, this falls back to a hard delete. - */ - softDeleteCurrentData(id: SourceKey, checkpointGreaterThan: bigint) { - if (!this.db.storageConfig.incrementalReprocessing) { - this.hardDeleteCurrentData(id); - return; - } - const op: mongo.AnyBulkWriteOperation = { - updateOne: { - filter: { _id: id }, - update: { - $set: { - data: EMPTY_DATA, - buckets: [] as CurrentDataDocumentV3['buckets'], - lookups: [] as CurrentDataDocumentV3['lookups'], - pending_delete: checkpointGreaterThan - } - }, - upsert: true } - }; - this.currentDataV3.push(op); - this.currentSize += 50; + }); } - upsertCurrentData(values: { - id: SourceKey; - data: bson.Binary | undefined; - buckets: CommonCurrentBucket[]; - lookups: CommonCurrentLookup[]; - }) { - if (this.db.storageConfig.incrementalReprocessing) { - const buckets = values.buckets.map((bucket) => { - if (!isCurrentBucketV3(bucket)) { - throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); - } - return bucket; - }); - const lookups = values.lookups.map((lookup) => { - if (!isRecordedLookupV3(lookup)) { - throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); - } - return lookup; - }); - const op: mongo.AnyBulkWriteOperation = { - updateOne: { - filter: { _id: values.id }, - update: { - $set: { - data: values.data, - buckets, - lookups - }, - $unset: { pending_delete: 1 } - }, - upsert: true - } - }; - this.currentDataV3.push(op); - } else { - const buckets = values.buckets.map((bucket) => ({ - bucket: bucket.bucket, - table: bucket.table, - id: bucket.id - })); - const lookups = values.lookups.map((lookup) => { - if (isRecordedLookupV3(lookup)) { - throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); - } - return lookup; - }); - const op: mongo.AnyBulkWriteOperation = { - updateOne: { - filter: { _id: values.id }, - update: { - $set: { - data: values.data, - buckets, - lookups - }, - $unset: { pending_delete: 1 } - }, - upsert: true - } - }; - this.currentDataV1.push(op); - } - this.currentSize += (values.data?.length() ?? 0) + 100; + protected flushBucketParameters() { + return this.bucketParameters.length > 0; } shouldFlushTransaction() { return ( this.currentSize >= MAX_TRANSACTION_BATCH_SIZE || this.bucketData.length >= MAX_TRANSACTION_DOC_COUNT || - this.currentDataV1.length + this.currentDataV3.length >= MAX_TRANSACTION_DOC_COUNT || + this.currentDataCount >= MAX_TRANSACTION_DOC_COUNT || this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT ); } @@ -442,43 +209,25 @@ export class PersistedBatch { flushedSomething = true; await db.bucket_data.bulkWrite(this.bucketData, { session, - // inserts only - order doesn't matter ordered: false }); } - if (this.bucketParameters.length > 0) { + if (this.flushBucketParameters()) { flushedSomething = true; await db.bucket_parameters.bulkWrite(this.bucketParameters, { session, - // inserts only - order doesn't matter ordered: false }); } - if (this.db.storageConfig.incrementalReprocessing) { - if (this.currentDataV3.length > 0) { - flushedSomething = true; - await db.v3_current_data.bulkWrite(this.currentDataV3, { - session, - // may update and delete data within the same batch - order matters - ordered: true - }); - } - } else { - if (this.currentDataV1.length > 0) { - flushedSomething = true; - await db.v1_current_data.bulkWrite(this.currentDataV1, { - session, - // may update and delete data within the same batch - order matters - ordered: true - }); - } + if (this.currentDataCount > 0) { + flushedSomething = true; + await this.flushCurrentData(session); } if (this.bucketStates.size > 0) { flushedSomething = true; await db.bucket_state.bulkWrite(this.getBucketStateUpdates(), { session, - // Per-bucket operation - order doesn't matter ordered: false }); } @@ -490,7 +239,7 @@ export class PersistedBatch { this.logger.info( `Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${ - this.currentDataV1.length + this.currentDataV3.length + this.currentDataCount } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`, { flushed: { @@ -498,7 +247,7 @@ export class PersistedBatch { size: this.currentSize, bucket_data_count: this.bucketData.length, parameter_data_count: this.bucketParameters.length, - current_data_count: this.currentDataV1.length + this.currentDataV3.length, + current_data_count: this.currentDataCount, replication_lag_seconds: replicationLag } } @@ -506,7 +255,7 @@ export class PersistedBatch { } else { this.logger.info( `Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${ - this.currentDataV1.length + this.currentDataV3.length + this.currentDataCount } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`, { flushed: { @@ -514,7 +263,7 @@ export class PersistedBatch { size: this.currentSize, bucket_data_count: this.bucketData.length, parameter_data_count: this.bucketParameters.length, - current_data_count: this.currentDataV1.length + this.currentDataV3.length + current_data_count: this.currentDataCount } } ); @@ -524,14 +273,13 @@ export class PersistedBatch { const stats = { bucketDataCount: this.bucketData.length, parameterDataCount: this.bucketParameters.length, - currentDataCount: this.currentDataV1.length + this.currentDataV3.length, + currentDataCount: this.currentDataCount, flushedAny: flushedSomething }; this.bucketData = []; this.bucketParameters = []; - this.currentDataV1 = []; - this.currentDataV3 = []; + this.resetCurrentData(); this.bucketStates.clear(); this.currentSize = 0; this.debugLastOpId = null; diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts new file mode 100644 index 000000000..b123d168a --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -0,0 +1,207 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import * as bson from 'bson'; + +import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatch.js'; +import { + PersistedBatch, + SaveBucketDataOptions, + SaveParameterDataOptions, + UpsertCurrentDataOptions +} from './PersistedBatch.js'; +import { + BucketParameterDocument, + CurrentBucket, + CurrentDataDocument, + SourceKey, + isCurrentBucketV3, + isRecordedLookupV3 +} from './models.js'; +import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; + +export class PersistedBatchV1 extends PersistedBatch { + currentData: mongo.AnyBulkWriteOperation[] = []; + + saveBucketData(options: SaveBucketDataOptions) { + const remaining_buckets = new Map(); + for (let bucket of options.before_buckets) { + if (isCurrentBucketV3(bucket)) { + throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); + } + remaining_buckets.set(currentBucketKey(bucket), bucket); + } + + const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); + + for (const evaluated of options.evaluated) { + const key = currentBucketKey({ + bucket: evaluated.bucket, + table: evaluated.table, + id: evaluated.id + }); + + const recordData = JSONBig.stringify(evaluated.data); + const checksum = utils.hashData(evaluated.table, evaluated.id, recordData); + if (recordData.length > MAX_ROW_SIZE) { + this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`); + continue; + } + + remaining_buckets.delete(key); + const byteEstimate = recordData.length + 200; + this.currentSize += byteEstimate; + + const op_id = options.op_seq.next(); + this.debugLastOpId = op_id; + + this.addBucketDataPut({ + op_id, + bucket: evaluated.bucket, + sourceTableId: options.table.id, + sourceKey: options.sourceKey, + table: evaluated.table, + rowId: evaluated.id, + checksum: BigInt(checksum), + data: recordData + }); + this.incrementBucket(evaluated.bucket, op_id, byteEstimate); + } + + for (let bucket of remaining_buckets.values()) { + const op_id = options.op_seq.next(); + this.debugLastOpId = op_id; + + this.addBucketDataRemove({ + op_id, + bucket: bucket.bucket, + sourceTableId: options.table.id, + sourceKey: options.sourceKey, + table: bucket.table, + rowId: bucket.id, + checksum: dchecksum + }); + this.currentSize += 200; + this.incrementBucket(bucket.bucket, op_id, 200); + } + } + + saveParameterData(data: SaveParameterDataOptions) { + const { sourceTable, sourceKey, evaluated } = data; + const remaining_lookups = new Map(); + + for (let lookup of data.existing_lookups) { + if (isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); + } + remaining_lookups.set(lookup.toString('base64'), lookup); + } + + for (let result of evaluated) { + const binLookup = storage.serializeLookup(result.lookup); + remaining_lookups.delete(binLookup.toString('base64')); + + const op_id = data.op_seq.next(); + this.debugLastOpId = op_id; + const values: BucketParameterDocument = { + _id: op_id, + key: { + g: this.group_id, + t: mongoTableId(sourceTable.id), + k: sourceKey + }, + lookup: binLookup, + bucket_parameters: result.bucketParameters + }; + this.bucketParameters.push({ + insertOne: { + document: values + } + }); + + this.currentSize += 200; + } + + for (let lookup of remaining_lookups.values()) { + const op_id = data.op_seq.next(); + this.debugLastOpId = op_id; + const values: BucketParameterDocument = { + _id: op_id, + key: { + g: this.group_id, + t: mongoTableId(sourceTable.id), + k: sourceKey + }, + lookup, + bucket_parameters: [] + }; + this.bucketParameters.push({ + insertOne: { + document: values + } + }); + + this.currentSize += 200; + } + } + + hardDeleteCurrentData(id: SourceKey) { + this.currentData.push({ + deleteOne: { + filter: { _id: id } + } + }); + this.currentSize += 50; + } + + softDeleteCurrentData(id: SourceKey, _checkpointGreaterThan: bigint) { + this.hardDeleteCurrentData(id); + } + + upsertCurrentData(values: UpsertCurrentDataOptions) { + const buckets = values.buckets.map((bucket) => { + if (isCurrentBucketV3(bucket)) { + throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); + } + return bucket; + }); + const lookups = values.lookups.map((lookup) => { + if (isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); + } + return lookup; + }); + + this.currentData.push({ + updateOne: { + filter: { _id: values.id }, + update: { + $set: { + data: values.data, + buckets, + lookups + }, + $unset: { pending_delete: 1 } + }, + upsert: true + } + }); + this.currentSize += (values.data?.length() ?? 0) + 100; + } + + protected get currentDataCount() { + return this.currentData.length; + } + + protected async flushCurrentData(session: mongo.ClientSession) { + await this.db.v1_current_data.bulkWrite(this.currentData, { + session, + ordered: true + }); + } + + protected resetCurrentData() { + this.currentData = []; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts new file mode 100644 index 000000000..a06138b27 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -0,0 +1,225 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatch.js'; +import { + PersistedBatch, + SaveBucketDataOptions, + SaveParameterDataOptions, + UpsertCurrentDataOptions +} from './PersistedBatch.js'; +import { + BucketParameterDocumentV3, + CurrentBucketV3, + CurrentDataDocumentV3, + RecordedLookupV3, + SourceKey, + isCurrentBucketV3, + isRecordedLookupV3 +} from './models.js'; +import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; + +export class PersistedBatchV3 extends PersistedBatch { + currentData: mongo.AnyBulkWriteOperation[] = []; + + saveBucketData(options: SaveBucketDataOptions) { + const remaining_buckets = new Map(); + for (let bucket of options.before_buckets) { + if (!isCurrentBucketV3(bucket)) { + throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); + } + remaining_buckets.set(currentBucketKey(bucket), bucket); + } + + const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); + + for (const evaluated of options.evaluated) { + const sourceDefinitionId = this.mapping.bucketSourceId(evaluated.source); + const key = currentBucketKey({ + def: sourceDefinitionId, + bucket: evaluated.bucket, + table: evaluated.table, + id: evaluated.id + }); + + const recordData = JSONBig.stringify(evaluated.data); + const checksum = utils.hashData(evaluated.table, evaluated.id, recordData); + if (recordData.length > MAX_ROW_SIZE) { + this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`); + continue; + } + + remaining_buckets.delete(key); + const byteEstimate = recordData.length + 200; + this.currentSize += byteEstimate; + + const op_id = options.op_seq.next(); + this.debugLastOpId = op_id; + + this.addBucketDataPut({ + op_id, + bucket: evaluated.bucket, + sourceTableId: options.table.id, + sourceKey: options.sourceKey, + table: evaluated.table, + rowId: evaluated.id, + checksum: BigInt(checksum), + data: recordData + }); + this.incrementBucket(evaluated.bucket, op_id, byteEstimate); + } + + for (let bucket of remaining_buckets.values()) { + const op_id = options.op_seq.next(); + this.debugLastOpId = op_id; + + this.addBucketDataRemove({ + op_id, + bucket: bucket.bucket, + sourceTableId: options.table.id, + sourceKey: options.sourceKey, + table: bucket.table, + rowId: bucket.id, + checksum: dchecksum + }); + this.currentSize += 200; + this.incrementBucket(bucket.bucket, op_id, 200); + } + } + + saveParameterData(data: SaveParameterDataOptions) { + const { sourceTable, sourceKey, evaluated } = data; + const remaining_lookups = new Map(); + + for (let lookup of data.existing_lookups) { + if (!isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + remaining_lookups.set(`${lookup.d}.${lookup.l.toString('base64')}`, lookup); + } + + for (let result of evaluated) { + const sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source); + const binLookup = storage.serializeLookup(result.lookup); + remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`); + + const op_id = data.op_seq.next(); + this.debugLastOpId = op_id; + const values: BucketParameterDocumentV3 = { + _id: op_id, + def: sourceDefinitionId, + key: { + g: this.group_id, + t: mongoTableId(sourceTable.id), + k: sourceKey + }, + lookup: binLookup, + bucket_parameters: result.bucketParameters + }; + this.bucketParameters.push({ + insertOne: { + document: values + } + }); + + this.currentSize += 200; + } + + for (let lookup of remaining_lookups.values()) { + const op_id = data.op_seq.next(); + this.debugLastOpId = op_id; + const values: BucketParameterDocumentV3 = { + _id: op_id, + def: lookup.d, + key: { + g: this.group_id, + t: mongoTableId(sourceTable.id), + k: sourceKey + }, + lookup: lookup.l, + bucket_parameters: [] + }; + this.bucketParameters.push({ + insertOne: { + document: values + } + }); + + this.currentSize += 200; + } + } + + hardDeleteCurrentData(id: SourceKey) { + this.currentData.push({ + deleteOne: { + filter: { _id: id } + } + }); + this.currentSize += 50; + } + + softDeleteCurrentData(id: SourceKey, checkpointGreaterThan: bigint) { + this.currentData.push({ + updateOne: { + filter: { _id: id }, + update: { + $set: { + data: EMPTY_DATA, + buckets: [] as CurrentDataDocumentV3['buckets'], + lookups: [] as CurrentDataDocumentV3['lookups'], + pending_delete: checkpointGreaterThan + } + }, + upsert: true + } + }); + this.currentSize += 50; + } + + upsertCurrentData(values: UpsertCurrentDataOptions) { + const buckets = values.buckets.map((bucket) => { + if (!isCurrentBucketV3(bucket)) { + throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); + } + return bucket; + }); + const lookups = values.lookups.map((lookup) => { + if (!isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + return lookup; + }); + + this.currentData.push({ + updateOne: { + filter: { _id: values.id }, + update: { + $set: { + data: values.data, + buckets, + lookups + }, + $unset: { pending_delete: 1 } + }, + upsert: true + } + }); + this.currentSize += (values.data?.length() ?? 0) + 100; + } + + protected get currentDataCount() { + return this.currentData.length; + } + + protected async flushCurrentData(session: mongo.ClientSession) { + await this.db.v3_current_data.bulkWrite(this.currentData, { + session, + ordered: true + }); + } + + protected resetCurrentData() { + this.currentData = []; + } +} From e9568ee236d882e86b4dd6987308ec30548666fe Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 12:07:45 +0200 Subject: [PATCH 04/93] Split out MongoBucketBatch implementations. --- .../implementation/MongoBucketBatch.ts | 130 ++++-------------- .../implementation/MongoBucketBatchV1.ts | 69 ++++++++++ .../implementation/MongoBucketBatchV3.ts | 88 ++++++++++++ .../implementation/MongoSyncBucketStorage.ts | 10 +- .../src/storage/storage-index.ts | 2 + .../test/src/storage_sync.test.ts | 31 +++-- 6 files changed, 213 insertions(+), 117 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index d664f59fd..3971a3760 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -1,5 +1,12 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { HydratedSyncRules, SqlEventDescriptor, SqliteRow, SqliteValue } from '@powersync/service-sync-rules'; +import { + EvaluatedParameters, + EvaluatedRow, + HydratedSyncRules, + SqlEventDescriptor, + SqliteRow, + SqliteValue +} from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { @@ -30,8 +37,6 @@ import { CommonCurrentBucket, CommonCurrentLookup, CommonCurrentDataDocument, - CurrentBucketV3, - RecordedLookupV3, SourceKey, SyncRuleDocument } from './models.js'; @@ -39,8 +44,6 @@ import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; -import { PersistedBatchV1 } from './PersistedBatchV1.js'; -import { PersistedBatchV3 } from './PersistedBatchV3.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; /** @@ -77,23 +80,23 @@ export interface MongoBucketBatchOptions { logger?: Logger; } -export class MongoBucketBatch +export abstract class MongoBucketBatch extends BaseObserver implements storage.BucketStorageBatch { - private logger: Logger; + protected logger: Logger; private readonly client: mongo.MongoClient; public readonly db: VersionedPowerSyncMongo; public readonly session: mongo.ClientSession; private readonly sync_rules: HydratedSyncRules; - private readonly group_id: number; + protected readonly group_id: number; private readonly slot_name: string; private readonly storeCurrentData: boolean; private readonly skipExistingRows: boolean; - private readonly mapping: BucketDefinitionMapping; + protected readonly mapping: BucketDefinitionMapping; private batch: OperationBatch | null = null; private write_checkpoint_batch: storage.CustomWriteCheckpointOptions[] = []; @@ -161,16 +164,20 @@ export class MongoBucketBatch return this.last_checkpoint_lsn; } - private createPersistedBatch(writtenSize: number): PersistedBatch { - if (this.db.storageConfig.incrementalReprocessing) { - return new PersistedBatchV3(this.db, this.group_id, this.mapping, writtenSize, { - logger: this.logger - }); - } - return new PersistedBatchV1(this.db, this.group_id, this.mapping, writtenSize, { - logger: this.logger - }); - } + protected abstract createPersistedBatch(writtenSize: number): PersistedBatch; + + protected abstract mapEvaluatedBuckets(evaluated: EvaluatedRow[]): CommonCurrentBucket[]; + + protected abstract mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CommonCurrentLookup[]; + + protected abstract createCurrentDataDocument( + id: SourceKey, + data: bson.Binary, + buckets: CommonCurrentBucket[], + lookups: CommonCurrentLookup[] + ): CommonCurrentDataDocument; + + protected abstract cleanupCurrentData(lastCheckpoint: bigint): Promise; async flush(options?: storage.BatchBucketFlushOptions): Promise { let result: storage.FlushedResult | null = null; @@ -521,25 +528,7 @@ export class MongoBucketBatch table: sourceTable, before_buckets: existing_buckets }); - if (this.db.storageConfig.incrementalReprocessing) { - new_buckets = evaluated.map((e) => { - const def = this.mapping.bucketSourceId(e.source); - return { - def, - bucket: e.bucket, - table: e.table, - id: e.id - } satisfies CurrentBucketV3; - }); - } else { - new_buckets = evaluated.map((e) => { - return { - bucket: e.bucket, - table: e.table, - id: e.id - }; - }); - } + new_buckets = this.mapEvaluatedBuckets(evaluated); } if (sourceTable.syncParameters) { @@ -572,16 +561,7 @@ export class MongoBucketBatch evaluated: paramEvaluated, existing_lookups }); - if (this.db.storageConfig.incrementalReprocessing) { - new_lookups = paramEvaluated.map((p) => { - const def = this.mapping.parameterLookupId(p.lookup.source); - return { d: def, l: storage.serializeLookup(p.lookup) } satisfies RecordedLookupV3; - }); - } else { - new_lookups = paramEvaluated.map((p) => { - return storage.serializeLookup(p.lookup); - }); - } + new_lookups = this.mapParameterLookups(paramEvaluated); } } @@ -597,45 +577,7 @@ export class MongoBucketBatch buckets: new_buckets, lookups: new_lookups }); - if (this.db.storageConfig.incrementalReprocessing) { - const buckets = new_buckets.map((bucket) => { - if (!('def' in bucket)) { - throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); - } - return bucket; - }); - const lookups = new_lookups.map((lookup) => { - if (lookup instanceof bson.Binary) { - throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); - } - return lookup; - }); - result = { - _id: after_key, - data: afterData!, - buckets, - lookups - }; - } else { - const buckets = new_buckets.map((bucket) => { - if ('def' in bucket) { - throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); - } - return bucket; - }); - const lookups = new_lookups.map((lookup) => { - if (!(lookup instanceof bson.Binary)) { - throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); - } - return lookup; - }); - result = { - _id: after_key, - data: afterData!, - buckets, - lookups - }; - } + result = this.createCurrentDataDocument(after_key, afterData!, new_buckets, new_lookups); } if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { @@ -924,25 +866,13 @@ export class MongoBucketBatch await this.db.notifyCheckpoint(); this.persisted_op = null; this.last_checkpoint_lsn = lsn; - if (this.db.storageConfig.incrementalReprocessing && newLastCheckpoint != null) { + if (newLastCheckpoint != null) { await this.cleanupCurrentData(newLastCheckpoint); } } return { checkpointBlocked, checkpointCreated }; } - private async cleanupCurrentData(lastCheckpoint: bigint) { - const result = await this.db.v3_current_data.deleteMany({ - '_id.g': this.group_id, - pending_delete: { $exists: true, $lte: lastCheckpoint } - }); - if (result.deletedCount > 0) { - this.logger.info( - `Cleaned up ${result.deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}` - ); - } - } - /** * Switch from processing -> active if relevant. * diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts new file mode 100644 index 000000000..ff259c028 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts @@ -0,0 +1,69 @@ +import * as bson from 'bson'; +import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; + +import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; +import { PersistedBatch } from './PersistedBatch.js'; +import { PersistedBatchV1 } from './PersistedBatchV1.js'; +import { + CommonCurrentBucket, + CommonCurrentLookup, + CurrentDataDocument, + SourceKey, + isCurrentBucketV3, + isRecordedLookupV3 +} from './models.js'; + +export class MongoBucketBatchV1 extends MongoBucketBatch { + constructor(options: MongoBucketBatchOptions) { + super(options); + } + + protected createPersistedBatch(writtenSize: number): PersistedBatch { + return new PersistedBatchV1(this.db, this.group_id, this.mapping, writtenSize, { + logger: this.logger + }); + } + + protected mapEvaluatedBuckets(evaluated: EvaluatedRow[]): CommonCurrentBucket[] { + return evaluated.map((entry) => ({ + bucket: entry.bucket, + table: entry.table, + id: entry.id + })); + } + + protected mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CommonCurrentLookup[] { + return paramEvaluated.map((entry) => storage.serializeLookup(entry.lookup)); + } + + protected createCurrentDataDocument( + id: SourceKey, + data: bson.Binary, + buckets: CommonCurrentBucket[], + lookups: CommonCurrentLookup[] + ): CurrentDataDocument { + const narrowedBuckets = buckets.map((bucket) => { + if (isCurrentBucketV3(bucket)) { + throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); + } + return bucket; + }); + const narrowedLookups = lookups.map((lookup) => { + if (isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); + } + return lookup; + }); + + return { + _id: id, + data, + buckets: narrowedBuckets, + lookups: narrowedLookups + }; + } + + protected async cleanupCurrentData(_lastCheckpoint: bigint): Promise {} +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts new file mode 100644 index 000000000..24ee8e0e5 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts @@ -0,0 +1,88 @@ +import * as bson from 'bson'; +import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; + +import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; +import { PersistedBatch } from './PersistedBatch.js'; +import { PersistedBatchV3 } from './PersistedBatchV3.js'; +import { + CommonCurrentBucket, + CommonCurrentLookup, + CurrentBucketV3, + CurrentDataDocumentV3, + RecordedLookupV3, + SourceKey, + isCurrentBucketV3, + isRecordedLookupV3 +} from './models.js'; + +export class MongoBucketBatchV3 extends MongoBucketBatch { + constructor(options: MongoBucketBatchOptions) { + super(options); + } + + protected createPersistedBatch(writtenSize: number): PersistedBatch { + return new PersistedBatchV3(this.db, this.group_id, this.mapping, writtenSize, { + logger: this.logger + }); + } + + protected mapEvaluatedBuckets(evaluated: EvaluatedRow[]): CommonCurrentBucket[] { + return evaluated.map((entry) => { + const def = this.mapping.bucketSourceId(entry.source); + return { + def, + bucket: entry.bucket, + table: entry.table, + id: entry.id + } satisfies CurrentBucketV3; + }); + } + + protected mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CommonCurrentLookup[] { + return paramEvaluated.map((entry) => { + const def = this.mapping.parameterLookupId(entry.lookup.source); + return { d: def, l: storage.serializeLookup(entry.lookup) } satisfies RecordedLookupV3; + }); + } + + protected createCurrentDataDocument( + id: SourceKey, + data: bson.Binary, + buckets: CommonCurrentBucket[], + lookups: CommonCurrentLookup[] + ): CurrentDataDocumentV3 { + const narrowedBuckets = buckets.map((bucket) => { + if (!isCurrentBucketV3(bucket)) { + throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); + } + return bucket; + }); + const narrowedLookups = lookups.map((lookup) => { + if (!isRecordedLookupV3(lookup)) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } + return lookup; + }); + + return { + _id: id, + data, + buckets: narrowedBuckets, + lookups: narrowedLookups + }; + } + + protected async cleanupCurrentData(lastCheckpoint: bigint): Promise { + const result = await this.db.v3_current_data.deleteMany({ + '_id.g': this.group_id, + pending_delete: { $exists: true, $lte: lastCheckpoint } + }); + if (result.deletedCount > 0) { + this.logger.info( + `Cleaned up ${result.deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}` + ); + } + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index c6f4d9c0c..52e27943a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -40,7 +40,8 @@ import { SourceKey, StorageConfig } from './models.js'; -import { MongoBucketBatch } from './MongoBucketBatch.js'; +import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; +import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; @@ -176,7 +177,7 @@ export class MongoSyncBucketStorage ); const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null; - const writer = new MongoBucketBatch({ + const batchOptions = { logger: options.logger, db: this.db, syncRules: this.sync_rules.parsed(options).hydratedSyncRules(), @@ -189,7 +190,10 @@ export class MongoSyncBucketStorage storeCurrentData: options.storeCurrentData, skipExistingRows: options.skipExistingRows ?? false, markRecordUnavailable: options.markRecordUnavailable - }); + }; + const writer = this.db.storageConfig.incrementalReprocessing + ? new MongoBucketBatchV3(batchOptions) + : new MongoBucketBatchV1(batchOptions); this.iterateListeners((cb) => cb.batchStarted?.(writer)); return writer; } diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index 4ba34a0cf..e15bb5e36 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -2,6 +2,8 @@ export * from './implementation/db.js'; export * from './implementation/BucketDefinitionMapping.js'; export * from './implementation/models.js'; export * from './implementation/MongoBucketBatch.js'; +export * from './implementation/MongoBucketBatchV1.js'; +export * from './implementation/MongoBucketBatchV3.js'; export * from './implementation/MongoIdSequence.js'; export * from './implementation/MongoPersistedSyncRules.js'; export * from './implementation/MongoPersistedSyncRulesContent.js'; diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index e0f032808..e749f5cf2 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -149,25 +149,28 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor ) ); const bucketStorage = factory.getInstance(syncRules); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); - await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => { - await batch.save({ - sourceTable: TEST_TABLE, - tag: storage.SaveOperationTag.INSERT, - after: { - id: 'shape-check', - description: 'shape' - }, - afterReplicaId: test_utils.rid('shape-check') - }); + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'shape-check', + description: 'shape' + }, + afterReplicaId: test_utils.rid('shape-check') }); + await writer.flush(); const mongoFactory = factory as MongoBucketStorage; - const currentData = (await mongoFactory.db.v3_current_data.findOne({})) as CurrentDataDocumentV3 | null; - expect(currentData?.buckets?.[0]?.def).toBeGreaterThan(0); + const currentData = await mongoFactory.db.v3_current_data.findOne({}); + const firstBucket: CurrentDataDocumentV3['buckets'][number] | undefined = currentData?.buckets[0]; + expect(firstBucket?.def).toBeGreaterThan(0); - const syncRule = (await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id })) as SyncRuleDocument | null; - expect(Object.keys(syncRule?.rule_mapping?.definitions ?? {})).not.toHaveLength(0); + const syncRule = await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id }); + const ruleMapping: SyncRuleDocument['rule_mapping'] | undefined = syncRule?.rule_mapping; + expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0); }); } From ede6515ef04b8a67b34f9c334a5a8d9f3e8a87bd Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 12:30:52 +0200 Subject: [PATCH 05/93] Resolve circular imports. --- .../src/storage/implementation/MongoBucketBatch.ts | 13 +------------ .../implementation/MongoBucketBatchShared.ts | 11 +++++++++++ .../src/storage/implementation/PersistedBatchV1.ts | 2 +- .../src/storage/implementation/PersistedBatchV3.ts | 2 +- .../src/storage/storage-index.ts | 3 --- 5 files changed, 14 insertions(+), 17 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 3971a3760..ae81231de 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -45,11 +45,7 @@ import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js' import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; - -/** - * 15MB - */ -export const MAX_ROW_SIZE = 15 * 1024 * 1024; +import { EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; // Currently, we can only have a single flush() at a time, since it locks the op_id sequence. // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex @@ -58,8 +54,6 @@ export const MAX_ROW_SIZE = 15 * 1024 * 1024; // In the future, we can investigate allowing multiple replication streams operating independently. const replicationMutex = new utils.Mutex(); -export const EMPTY_DATA = new bson.Binary(bson.serialize({})); - export interface MongoBucketBatchOptions { db: VersionedPowerSyncMongo; syncRules: HydratedSyncRules; @@ -1210,8 +1204,3 @@ export abstract class MongoBucketBatch ); } } - -export function currentBucketKey(b: CommonCurrentBucket) { - const prefix = 'def' in b ? `${b.def}:` : ''; - return `${prefix}${b.bucket}/${b.table}/${b.id}`; -} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts new file mode 100644 index 000000000..1477b8711 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts @@ -0,0 +1,11 @@ +import * as bson from 'bson'; +import { CommonCurrentBucket } from './models.js'; + +export const MAX_ROW_SIZE = 15 * 1024 * 1024; + +export const EMPTY_DATA = new bson.Binary(bson.serialize({})); + +export function currentBucketKey(bucket: CommonCurrentBucket) { + const prefix = 'def' in bucket ? `${bucket.def}:` : ''; + return `${prefix}${bucket.bucket}/${bucket.table}/${bucket.id}`; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index b123d168a..535ae1c43 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -4,7 +4,7 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import * as bson from 'bson'; -import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatch.js'; +import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; import { PersistedBatch, SaveBucketDataOptions, diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index a06138b27..5448607ca 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -2,7 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { JSONBig } from '@powersync/service-jsonbig'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; -import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatch.js'; +import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; import { PersistedBatch, SaveBucketDataOptions, diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index e15bb5e36..75e1f323b 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -1,9 +1,6 @@ export * from './implementation/db.js'; export * from './implementation/BucketDefinitionMapping.js'; export * from './implementation/models.js'; -export * from './implementation/MongoBucketBatch.js'; -export * from './implementation/MongoBucketBatchV1.js'; -export * from './implementation/MongoBucketBatchV3.js'; export * from './implementation/MongoIdSequence.js'; export * from './implementation/MongoPersistedSyncRules.js'; export * from './implementation/MongoPersistedSyncRulesContent.js'; From 5c76a2db6871d321374397f36eda23e818cac994 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 13:11:37 +0200 Subject: [PATCH 06/93] Back to old bucket names for now. --- .../src/storage/implementation/MongoPersistedSyncRules.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index ba5b4eb5e..547c35bc3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -24,7 +24,9 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules { private readonly storageConfig: StorageConfig ) { if (this.mapping != null && this.storageConfig.incrementalReprocessing) { - this.hydrationState = new MongoHydrationState(this.mapping); + // FIXME: Recheck bucket name generation again when we get to merging sync config versions. + // this.hydrationState = new MongoHydrationState(this.mapping); + this.hydrationState = versionedHydrationState(this.id); } else if ( !this.sync_rules.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && !this.storageConfig.versionedBuckets From c64020a43abadb50384989db742a3fab07a4f39d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 13:11:44 +0200 Subject: [PATCH 07/93] Fix type check. --- .../module-mongodb-storage/src/storage/implementation/models.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index c020c0a8b..3de1a23e1 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -329,5 +329,5 @@ export function isCurrentBucketV3(bucket: CommonCurrentBucket): bucket is Curren } export function isRecordedLookupV3(lookup: CommonCurrentLookup): lookup is RecordedLookupV3 { - return !(lookup instanceof bson.Binary); + return typeof lookup === 'object' && lookup != null && 'd' in lookup && 'l' in lookup; } From de659ba795f9b14b4dc1e7d2f19458bf4be551ff Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 13:25:10 +0200 Subject: [PATCH 08/93] nullable CurrentDataDocumentV3.data. --- .../src/storage/implementation/MongoBucketBatch.ts | 10 +++++----- .../src/storage/implementation/PersistedBatch.ts | 2 +- .../src/storage/implementation/PersistedBatchV1.ts | 4 ++-- .../src/storage/implementation/PersistedBatchV3.ts | 14 +++++++------- .../src/storage/implementation/models.ts | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index ae81231de..69c2ed13e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -314,7 +314,7 @@ export abstract class MongoBucketBatch if (nextData != null) { // Update our current_data and size cache current_data_lookup.set(op.internalAfterKey!, nextData); - sizes?.set(op.internalAfterKey!, nextData.data.length()); + sizes?.set(op.internalAfterKey!, nextData.data?.length() ?? 0); } if (persistedBatch!.shouldFlushTransaction()) { @@ -399,8 +399,8 @@ export abstract class MongoBucketBatch } else { existing_buckets = result.buckets; existing_lookups = result.lookups; - if (this.storeCurrentData) { - const data = deserializeBson((result.data as mongo.Binary).buffer) as SqliteRow; + if (this.storeCurrentData && result.data != null) { + const data = deserializeBson(result.data.buffer) as SqliteRow; after = storage.mergeToast(after!, data); } } @@ -422,9 +422,9 @@ export abstract class MongoBucketBatch } } - let afterData: bson.Binary | undefined; + let afterData: bson.Binary | null = null; if (afterId != null && !this.storeCurrentData) { - afterData = EMPTY_DATA; + afterData = null; } else if (afterId != null) { try { // This will fail immediately if the record is > 16MB. diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index 3a96db749..5e6a38e90 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -56,7 +56,7 @@ export interface SaveParameterDataOptions { export interface UpsertCurrentDataOptions { id: SourceKey; - data: bson.Binary | undefined; + data: bson.Binary | null; buckets: CommonCurrentBucket[]; lookups: CommonCurrentLookup[]; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index 535ae1c43..5e505e723 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -4,7 +4,7 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import * as bson from 'bson'; -import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; +import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; import { PersistedBatch, SaveBucketDataOptions, @@ -178,7 +178,7 @@ export class PersistedBatchV1 extends PersistedBatch { filter: { _id: values.id }, update: { $set: { - data: values.data, + data: values.data ?? EMPTY_DATA, buckets, lookups }, diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 5448607ca..427b5dc8d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -1,8 +1,9 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { JSONBig } from '@powersync/service-jsonbig'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; -import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; +import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; import { PersistedBatch, SaveBucketDataOptions, @@ -13,12 +14,11 @@ import { BucketParameterDocumentV3, CurrentBucketV3, CurrentDataDocumentV3, - RecordedLookupV3, - SourceKey, isCurrentBucketV3, - isRecordedLookupV3 + isRecordedLookupV3, + RecordedLookupV3, + SourceKey } from './models.js'; -import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; export class PersistedBatchV3 extends PersistedBatch { currentData: mongo.AnyBulkWriteOperation[] = []; @@ -165,7 +165,7 @@ export class PersistedBatchV3 extends PersistedBatch { filter: { _id: id }, update: { $set: { - data: EMPTY_DATA, + data: null, buckets: [] as CurrentDataDocumentV3['buckets'], lookups: [] as CurrentDataDocumentV3['lookups'], pending_delete: checkpointGreaterThan diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 3de1a23e1..2ac8df07e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -49,7 +49,7 @@ export interface RecordedLookupV3 { export interface CurrentDataDocumentV3 { _id: SourceKey; - data: bson.Binary; + data: bson.Binary | null; buckets: CurrentBucketV3[]; lookups: RecordedLookupV3[]; /** From 70db4e96a09a4801a6e7c829eff59584dba5859a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 15:59:01 +0200 Subject: [PATCH 09/93] Use string ids. --- .../implementation/BucketDefinitionMapping.ts | 23 +++++++++++-------- .../implementation/MongoBucketBatchV3.ts | 2 +- .../implementation/MongoPersistedSyncRules.ts | 4 ++-- .../implementation/PersistedBatchV3.ts | 6 ++--- .../src/storage/implementation/models.ts | 11 +++++---- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts index f066a2013..4656b06be 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -2,9 +2,12 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { BucketDataSource, ParameterIndexLookupCreator, SyncConfigWithErrors } from '@powersync/service-sync-rules'; import { SyncRuleDocument } from './models.js'; +export type BucketDefinitionId = string; +export type ParameterIndexId = string; + export class BucketDefinitionMapping { static fromSyncRules(doc: Pick): BucketDefinitionMapping { - return new BucketDefinitionMapping(doc.rule_mapping?.definitions ?? {}, doc.rule_mapping?.parameter_lookups ?? {}); + return new BucketDefinitionMapping(doc.rule_mapping?.definitions ?? {}, doc.rule_mapping?.parameter_indexes ?? {}); } static fromParsedSyncRules(syncRules: SyncConfigWithErrors): BucketDefinitionMapping { @@ -13,25 +16,25 @@ export class BucketDefinitionMapping { .map((source) => `${source.defaultLookupScope.lookupName}#${source.defaultLookupScope.queryId}`) .sort(); - const definitions: Record = {}; - const parameterLookups: Record = {}; + const definitions: Record = {}; + const parameterLookups: Record = {}; for (const [index, uniqueName] of definitionNames.entries()) { - definitions[uniqueName] = index + 1; + definitions[uniqueName] = (index + 1).toString(16); } for (const [index, key] of parameterKeys.entries()) { - parameterLookups[key] = index + 1; + parameterLookups[key] = (index + 1).toString(16); } return new BucketDefinitionMapping(definitions, parameterLookups); } constructor( - private definitions: Record = {}, - private parameterLookupMapping: Record = {} + private definitions: Record = {}, + private parameterLookupMapping: Record = {} ) {} - bucketSourceId(source: BucketDataSource): number { + bucketSourceId(source: BucketDataSource): BucketDefinitionId { const defId = this.definitions[source.uniqueName]; if (defId == null) { throw new ServiceAssertionError(`No mapping found for bucket source ${source.uniqueName}`); @@ -39,7 +42,7 @@ export class BucketDefinitionMapping { return defId; } - parameterLookupId(source: ParameterIndexLookupCreator): number { + parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId { const key = `${source.defaultLookupScope.lookupName}#${source.defaultLookupScope.queryId}`; const defId = this.parameterLookupMapping[key]; if (defId == null) { @@ -51,7 +54,7 @@ export class BucketDefinitionMapping { serialize(): NonNullable { return { definitions: { ...this.definitions }, - parameter_lookups: { ...this.parameterLookupMapping } + parameter_indexes: { ...this.parameterLookupMapping } }; } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts index 24ee8e0e5..5b593dae6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts @@ -43,7 +43,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { protected mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CommonCurrentLookup[] { return paramEvaluated.map((entry) => { const def = this.mapping.parameterLookupId(entry.lookup.source); - return { d: def, l: storage.serializeLookup(entry.lookup) } satisfies RecordedLookupV3; + return { i: def, l: storage.serializeLookup(entry.lookup) } satisfies RecordedLookupV3; }); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index 547c35bc3..c3b4870a0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -48,7 +48,7 @@ class MongoHydrationState implements HydrationState { getBucketSourceScope(source: BucketDataSource): BucketDataScope { const defId = this.mapping.bucketSourceId(source); return { - bucketPrefix: defId.toString(16), + bucketPrefix: defId, source }; } @@ -56,7 +56,7 @@ class MongoHydrationState implements HydrationState { getParameterIndexLookupScope(source: ParameterIndexLookupCreator) { const defId = this.mapping.parameterLookupId(source); return { - lookupName: defId.toString(16), + lookupName: defId, queryId: '', source }; diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 427b5dc8d..8fb17aa7b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -96,7 +96,7 @@ export class PersistedBatchV3 extends PersistedBatch { if (!isRecordedLookupV3(lookup)) { throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); } - remaining_lookups.set(`${lookup.d}.${lookup.l.toString('base64')}`, lookup); + remaining_lookups.set(`${lookup.i}.${lookup.l.toString('base64')}`, lookup); } for (let result of evaluated) { @@ -108,7 +108,7 @@ export class PersistedBatchV3 extends PersistedBatch { this.debugLastOpId = op_id; const values: BucketParameterDocumentV3 = { _id: op_id, - def: sourceDefinitionId, + index: sourceDefinitionId, key: { g: this.group_id, t: mongoTableId(sourceTable.id), @@ -131,7 +131,7 @@ export class PersistedBatchV3 extends PersistedBatch { this.debugLastOpId = op_id; const values: BucketParameterDocumentV3 = { _id: op_id, - def: lookup.d, + index: lookup.i, key: { g: this.group_id, t: mongoTableId(sourceTable.id), diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 2ac8df07e..1b05d836e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -3,6 +3,7 @@ import { SqliteJsonValue } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { event_types } from '@powersync/service-types'; import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; +import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; /** * Replica id uniquely identifying a row on the source database. @@ -39,11 +40,11 @@ export interface CurrentDataDocument { } export interface CurrentBucketV3 extends CurrentBucket { - def: number; + def: BucketDefinitionId; } export interface RecordedLookupV3 { - d: number; + i: ParameterIndexId; l: bson.Binary; } @@ -74,7 +75,7 @@ export interface BucketParameterDocument { } export interface BucketParameterDocumentV3 extends BucketParameterDocument { - def: number; + index: ParameterIndexId; } export interface BucketDataDocument { @@ -233,8 +234,8 @@ export interface SyncRuleDocument { content: string; serialized_plan?: SerializedSyncPlan | null; rule_mapping?: { - definitions: Record; - parameter_lookups: Record; + definitions: Record; + parameter_indexes: Record; }; lock?: { From 45f14e37850682e0c0ef0e3aa32cd1c826201232 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 16:57:11 +0200 Subject: [PATCH 10/93] Split collections for bucket_data. --- .../src/storage/MongoBucketStorage.ts | 19 +- .../storage/implementation/MongoChecksums.ts | 129 ++++- .../storage/implementation/MongoCompactor.ts | 440 ++++++++++++------ .../implementation/MongoSyncBucketStorage.ts | 173 ++++++- .../storage/implementation/PersistedBatch.ts | 98 ++-- .../implementation/PersistedBatchV1.ts | 18 + .../implementation/PersistedBatchV3.ts | 29 +- .../src/storage/implementation/db.ts | 53 ++- .../src/storage/implementation/models.ts | 73 ++- .../module-mongodb-storage/src/utils/util.ts | 4 +- .../test/src/storage_sync.test.ts | 31 +- 11 files changed, 815 insertions(+), 252 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 8cfbdb106..78830c549 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -312,7 +312,6 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { }; } const operations_aggregate = await this.db.bucket_data - .aggregate([ { $collStats: { @@ -322,6 +321,20 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ]) .toArray() .catch(ignoreNotExisting); + const v3_operation_aggregates = await Promise.all( + (await this.db.listBucketDataCollectionsV3()).map((collection) => + collection + .aggregate([ + { + $collStats: { + storageStats: {} + } + } + ]) + .toArray() + .catch(ignoreNotExisting) + ) + ); const parameters_aggregate = await this.db.bucket_parameters .aggregate([ @@ -356,7 +369,9 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { .toArray() .catch(ignoreNotExisting); return { - operations_size_bytes: Number(operations_aggregate[0].storageStats.size), + operations_size_bytes: + Number(operations_aggregate[0].storageStats.size) + + v3_operation_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0), parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size), replication_size_bytes: Number(v1_replication_aggregate[0].storageStats.size) + Number(v3_replication_aggregate[0].storageStats.size) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index 0eb2c99e3..2d9cc73ea 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -1,4 +1,5 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; import { addPartialChecksums, bson, @@ -14,6 +15,7 @@ import { PartialOrFullChecksum } from '@powersync/service-core'; import { VersionedPowerSyncMongo } from './db.js'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { StorageConfig } from './models.js'; /** @@ -31,6 +33,7 @@ export interface MongoChecksumOptions { operationBatchLimit?: number; storageConfig: StorageConfig; + mapping: BucketDefinitionMapping; } const DEFAULT_BUCKET_BATCH_LIMIT = 200; @@ -48,6 +51,7 @@ const DEFAULT_OPERATION_BATCH_LIMIT = 50_000; export class MongoChecksums { private _cache: ChecksumCache | undefined; private readonly storageConfig: StorageConfig; + private readonly mapping: BucketDefinitionMapping; constructor( private db: VersionedPowerSyncMongo, @@ -55,6 +59,7 @@ export class MongoChecksums { private options: MongoChecksumOptions ) { this.storageConfig = options.storageConfig; + this.mapping = options.mapping; } /** @@ -200,6 +205,104 @@ export class MongoChecksums { * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this. */ private async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + if (this.storageConfig.incrementalReprocessing) { + const results = new Map(); + const requestsByDefinition = new Map(); + const fallbackRequests: FetchPartialBucketChecksum[] = []; + for (const request of batch) { + if (!isBucketSourceLike(request.source)) { + fallbackRequests.push(request); + continue; + } + const definitionId = this.mapping.bucketSourceId(request.source); + const existing = requestsByDefinition.get(definitionId) ?? []; + existing.push(request); + requestsByDefinition.set(definitionId, existing); + } + + for (const [definitionId, requests] of requestsByDefinition.entries()) { + const groupResults = await this.computePartialChecksumsForCollection( + requests, + this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, + (request) => ({ + _id: { + $gt: { + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + b: request.bucket, + o: request.end + } + } + }) + ); + for (const checksum of groupResults.values()) { + results.set(checksum.bucket, checksum); + } + } + + if (fallbackRequests.length > 0) { + const collections = await this.db.listBucketDataCollectionsV3(this.group_id); + for (const request of fallbackRequests) { + let merged: PartialOrFullChecksum | null = null; + for (const collection of collections) { + const groupResults = await this.computePartialChecksumsForCollection( + [request], + collection as unknown as mongo.Collection, + (entry) => ({ + _id: { + $gt: { + b: entry.bucket, + o: entry.start ?? new bson.MinKey() + }, + $lte: { + b: entry.bucket, + o: entry.end + } + } + }) + ); + merged = addPartialChecksums(request.bucket, merged, groupResults.get(request.bucket) ?? null); + } + results.set( + request.bucket, + merged ?? + (request.start == null + ? { bucket: request.bucket, count: 0, checksum: 0 } + : { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }) + ); + } + } + + return results; + } + + return this.computePartialChecksumsForCollection( + batch, + this.db.bucket_data as unknown as mongo.Collection, + (request) => ({ + _id: { + $gt: { + g: this.group_id, + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + g: this.group_id, + b: request.bucket, + o: request.end + } + } + }) + ); + } + + private async computePartialChecksumsForCollection( + batch: FetchPartialBucketChecksum[], + collection: mongo.Collection, + createFilter: (request: FetchPartialBucketChecksum) => any + ): Promise { const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT; // Map requests by bucket. We adjust this as we get partial results. @@ -211,23 +314,7 @@ export class MongoChecksums { const partialChecksums = new Map(); while (requests.size > 0) { - const filters: any[] = []; - for (let request of requests.values()) { - filters.push({ - _id: { - $gt: { - g: this.group_id, - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - g: this.group_id, - b: request.bucket, - o: request.end - } - } - }); - } + const filters = Array.from(requests.values(), createFilter); // Historically, checksum may be stored as 'int' or 'double'. // More recently, this should be a 'long'. @@ -243,7 +330,7 @@ export class MongoChecksums { // Returns: B[3-10], C[1-4] // 3. Query: C[5-end] // Returns: C[5-10] - const aggregate = await this.db.bucket_data + const aggregate = await collection .aggregate( [ { @@ -339,6 +426,12 @@ export class MongoChecksums { } } +function isBucketSourceLike( + source: FetchPartialBucketChecksum['source'] +): source is NonNullable { + return source != null && typeof source == 'object' && 'uniqueName' in source; +} + /** * Convert output of the $group stage into a checksum. */ diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index d957f17b1..78553db1f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -10,7 +10,17 @@ import { } from '@powersync/service-core'; import { VersionedPowerSyncMongo } from './db.js'; -import { BucketDataDocument, BucketDataKey, BucketStateDocument } from './models.js'; +import { BucketDefinitionId } from './BucketDefinitionMapping.js'; +import { + BucketDataDocumentV1, + BucketDataDocumentV3, + BucketStateDocument, + LEGACY_BUCKET_DATA_DEFINITION_ID, + TaggedBucketDataDocument, + bucketDataDocumentToTagged, + taggedBucketDataDocumentToV1, + taggedBucketDataDocumentToV3 +} from './models.js'; import { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; import { cacheKey } from './OperationBatch.js'; @@ -53,6 +63,22 @@ interface CurrentBucketState { opBytes: number; } +type CompactBucketDataDocument = Pick< + TaggedBucketDataDocument, + '_id' | 'def' | 'op' | 'table' | 'row_id' | 'source_table' | 'source_key' | 'checksum' | 'target_op' +> & { + size: number | bigint; +}; + +type CompactClearBucketDataDocument = Pick; +type BucketDataCollectionDocument = BucketDataDocumentV1 | BucketDataDocumentV3; +type BucketDataClearProjection = { + _id: BucketDataCollectionDocument['_id']; + op: CompactClearBucketDataDocument['op']; + checksum: bigint; + target_op?: bigint | null; +}; + /** * Additional options, primarily for testing. */ @@ -69,8 +95,10 @@ const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; const DEFAULT_MEMORY_LIMIT_MB = 64; export class MongoCompactor { - private updates: mongo.AnyBulkWriteOperation[] = []; + private updates: mongo.AnyBulkWriteOperation[] = []; private bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; + private activeBucketDataCollection: mongo.Collection | null = null; + private activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; private idLimitBytes: number; private moveBatchLimit: number; @@ -138,6 +166,12 @@ export class MongoCompactor { private async compactSingleBucket(bucket: string) { const idLimitBytes = this.idLimitBytes; + const bucketCollection = await this.getBucketDataCollection(bucket); + if (bucketCollection == null) { + return; + } + this.activeBucketDataCollection = bucketCollection.collection; + this.activeBucketDefinitionId = bucketCollection.definitionId; let currentState: CurrentBucketState = { bucket, @@ -152,154 +186,156 @@ export class MongoCompactor { }; // Constant lower bound - const lowerBound: BucketDataKey = { - g: this.group_id, - b: bucket, - o: new mongo.MinKey() as any - }; + const lowerBound = this.bucketDataKey({ + _id: { b: bucket, o: new mongo.MinKey() as any }, + def: bucketCollection.definitionId + }); // Upper bound is adjusted for each batch - let upperBound: BucketDataKey = { - g: this.group_id, - b: bucket, - o: new mongo.MaxKey() as any - }; + let upperBound = this.bucketDataKey({ + _id: { b: bucket, o: new mongo.MaxKey() as any }, + def: bucketCollection.definitionId + }); - while (!this.signal?.aborted) { - // Query one batch at a time, to avoid cursor timeouts - const cursor = this.db.bucket_data.aggregate( - [ - { - $match: { - _id: { - $gte: lowerBound, - $lt: upperBound + try { + while (!this.signal?.aborted) { + // Query one batch at a time, to avoid cursor timeouts + const cursor = bucketCollection.collection.aggregate( + [ + { + $match: { + _id: { + $gte: lowerBound, + $lt: upperBound + } + } + }, + { $sort: { _id: -1 } }, + { $limit: this.moveBatchQueryLimit }, + { + $project: { + _id: 1, + op: 1, + table: 1, + row_id: 1, + source_table: 1, + source_key: 1, + checksum: 1, + size: { $bsonSize: '$$ROOT' } } } - }, - { $sort: { _id: -1 } }, - { $limit: this.moveBatchQueryLimit }, + ], { - $project: { - _id: 1, - op: 1, - table: 1, - row_id: 1, - source_table: 1, - source_key: 1, - checksum: 1, - size: { $bsonSize: '$$ROOT' } - } + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: this.moveBatchQueryLimit + 1 } - ], - { - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: this.moveBatchQueryLimit + 1 - } - ); - // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. - // Instead, we load up to the limit. - const batch = await cursor.toArray(); + ); + // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. + // Instead, we load up to the limit. + const rawBatch = await cursor.toArray(); + const batch = rawBatch.map((document) => this.tagBucketDataDocument(document, bucketCollection.definitionId)); - if (batch.length == 0) { - // We've reached the end - break; - } + if (batch.length == 0) { + // We've reached the end + break; + } - // Set upperBound for the next batch - upperBound = batch[batch.length - 1]._id; + // Set upperBound for the next batch + upperBound = this.bucketDataKey(batch[batch.length - 1]); - for (let doc of batch) { - if (doc._id.o > this.maxOpId) { - continue; - } + for (let doc of batch) { + if (doc._id.o > this.maxOpId) { + continue; + } - currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum)); - currentState.opCount += 1; - - let isPersistentPut = doc.op == 'PUT'; - - currentState.opBytes += Number(doc.size); - if (doc.op == 'REMOVE' || doc.op == 'PUT') { - const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; - const targetOp = currentState.seen.get(key); - if (targetOp) { - // Will convert to MOVE, so don't count as PUT - isPersistentPut = false; - - this.updates.push({ - updateOne: { - filter: { - _id: doc._id - }, - update: { - $set: { - op: 'MOVE', - target_op: targetOp - }, - $unset: { - source_table: 1, - source_key: 1, - table: 1, - row_id: 1, - data: 1 + currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum)); + currentState.opCount += 1; + + let isPersistentPut = doc.op == 'PUT'; + + currentState.opBytes += Number(doc.size); + if (doc.op == 'REMOVE' || doc.op == 'PUT') { + const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; + const targetOp = currentState.seen.get(key); + if (targetOp) { + // Will convert to MOVE, so don't count as PUT + isPersistentPut = false; + + this.updates.push({ + updateOne: { + filter: { _id: this.bucketDataKey(doc) }, + update: { + $set: { + op: 'MOVE', + target_op: targetOp + }, + $unset: { + source_table: 1, + source_key: 1, + table: 1, + row_id: 1, + data: 1 + } } } - } - }); + }); - currentState.opBytes += 200 - Number(doc.size); // TODO: better estimate for this - } else { - if (currentState.trackingSize >= idLimitBytes) { - // Reached memory limit. - // Keep the highest seen values in this case. + currentState.opBytes += 200 - Number(doc.size); // TODO: better estimate for this } else { - // flatstr reduces the memory usage by flattening the string - currentState.seen.set(utils.flatstr(key), doc._id.o); - // length + 16 for the string - // 24 for the bigint - // 50 for map overhead - // 50 for additional overhead - currentState.trackingSize += key.length + 140; + if (currentState.trackingSize >= idLimitBytes) { + // Reached memory limit. + // Keep the highest seen values in this case. + } else { + // flatstr reduces the memory usage by flattening the string + currentState.seen.set(utils.flatstr(key), doc._id.o); + // length + 16 for the string + // 24 for the bigint + // 50 for map overhead + // 50 for additional overhead + currentState.trackingSize += key.length + 140; + } } } - } - if (isPersistentPut) { - currentState.lastNotPut = null; - currentState.opsSincePut = 0; - } else if (doc.op != 'CLEAR') { - if (currentState.lastNotPut == null) { - currentState.lastNotPut = doc._id.o; + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (doc.op != 'CLEAR') { + if (currentState.lastNotPut == null) { + currentState.lastNotPut = doc._id.o; + } + currentState.opsSincePut += 1; } - currentState.opsSincePut += 1; - } - if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { - await this.flush(); + if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { + await this.flush(); + } } + + logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); } - logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); - } + // Free memory before clearing bucket + currentState.seen.clear(); + if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { + logger.info( + `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + // Need flush() before clear() + await this.flush(); + await this.clearBucket(currentState); + } - // Free memory before clearing bucket - currentState.seen.clear(); - if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { - logger.info( - `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` - ); - // Need flush() before clear() + // Do this _after_ clearBucket so that we have accurate counts. + this.updateBucketChecksums(currentState); + + // Need another flush after updateBucketChecksums() await this.flush(); - await this.clearBucket(currentState); + } finally { + this.activeBucketDataCollection = null; + this.activeBucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; } - - // Do this _after_ clearBucket so that we have accurate counts. - this.updateBucketChecksums(currentState); - - // Need another flush after updateBucketChecksums() - await this.flush(); } /** @@ -346,7 +382,10 @@ export class MongoCompactor { private async flush() { if (this.updates.length > 0) { logger.info(`Compacting ${this.updates.length} ops`); - await this.db.bucket_data.bulkWrite(this.updates, { + if (this.activeBucketDataCollection == null) { + throw new ServiceAssertionError('No bucket_data collection selected for compaction'); + } + await this.activeBucketDataCollection.bulkWrite(this.updates, { // Order is not important. // Since checksums are not affected, these operations can happen in any order, // and it's fine if the operations are partially applied. @@ -374,19 +413,18 @@ export class MongoCompactor { private async clearBucket(currentState: CurrentBucketState) { const bucket = currentState.bucket; const clearOp = currentState.lastNotPut!; + const bucketCollection = this.activeBucketDataCollection; + if (bucketCollection == null) { + throw new ServiceAssertionError('No bucket_data collection selected for compaction'); + } const opFilter = { _id: { - $gte: { - g: this.group_id, - b: bucket, - o: new mongo.MinKey() as any - }, - $lte: { - g: this.group_id, - b: bucket, - o: clearOp - } + $gte: this.bucketDataKey({ + _id: { b: bucket, o: new mongo.MinKey() as any }, + def: this.activeBucketDefinitionId + }), + $lte: this.bucketDataKey({ _id: { b: bucket, o: clearOp }, def: this.activeBucketDefinitionId }) } }; @@ -400,7 +438,7 @@ export class MongoCompactor { // We need a transaction per batch to make sure checksums stay consistent. await session.withTransaction( async () => { - const query = this.db.bucket_data.find(opFilter, { + const query = bucketCollection.find(opFilter as any, { session, sort: { _id: 1 }, projection: { @@ -412,14 +450,19 @@ export class MongoCompactor { limit: this.clearBatchLimit }); let checksum = 0; - let lastOpId: BucketDataKey | null = null; + let lastOp: CompactClearBucketDataDocument | null = null; let targetOp: bigint | null = null; let gotAnOp = false; let numberOfOpsToClear = 0; - for await (let op of query.stream()) { + for await (let rawOp of query.stream()) { + const op = this.tagClearBucketDataDocument( + rawOp as unknown as BucketDataClearProjection, + this.activeBucketDefinitionId + ); + if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') { checksum = utils.addChecksums(checksum, Number(op.checksum)); - lastOpId = op._id; + lastOp = op; numberOfOpsToClear += 1; if (op.op != 'CLEAR') { gotAnOp = true; @@ -431,7 +474,7 @@ export class MongoCompactor { } } else { throw new ReplicationAssertionError( - `Unexpected ${op.op} operation at ${op._id.g}:${op._id.b}:${op._id.o}` + `Unexpected ${op.op} operation at ${this.formatBucketDataKey(op._id as unknown as mongo.Document)}` ); } } @@ -440,30 +483,30 @@ export class MongoCompactor { return; } - logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOpId?.o}`); - await this.db.bucket_data.deleteMany( + logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?._id.o}`); + await bucketCollection.deleteMany( { _id: { - $gte: { - g: this.group_id, - b: bucket, - o: new mongo.MinKey() as any - }, - $lte: lastOpId! + $gte: this.bucketDataKey({ + _id: { b: bucket, o: new mongo.MinKey() as any }, + def: this.activeBucketDefinitionId + }), + $lte: this.bucketDataKey(lastOp!) } - }, - { session } + } as any, + { session } as any ); - await this.db.bucket_data.insertOne( - { - _id: lastOpId!, + await bucketCollection.insertOne( + this.collectionBucketDataDocument({ + def: this.activeBucketDefinitionId, + _id: lastOp!._id, op: 'CLEAR', checksum: BigInt(checksum), data: null, target_op: targetOp - }, - { session } + }) as unknown as mongo.OptionalId, + { session } as any ); opCountDiff = -numberOfOpsToClear + 1; @@ -705,4 +748,91 @@ export class MongoCompactor { await this.flush(); } + + private bucketDataKey(document: Pick) { + if (this.db.storageConfig.incrementalReprocessing) { + return taggedBucketDataDocumentToV3({ + def: document.def, + _id: document._id, + op: 'CLEAR', + checksum: 0n, + data: null + })._id; + } + + return taggedBucketDataDocumentToV1(this.group_id, { + def: document.def, + _id: document._id, + op: 'CLEAR', + checksum: 0n, + data: null + })._id; + } + + private async getBucketDataCollection( + bucket: string + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { + if (!this.db.storageConfig.incrementalReprocessing) { + return { + collection: this.db.v1_bucket_data as unknown as mongo.Collection, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID + }; + } + + for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { + const existing = await collection.findOne( + { '_id.b': bucket }, + { projection: { _id: 1 }, maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } + ); + if (existing != null) { + const definitionId = collection.collectionName.replace(`bucket_data_${this.group_id}_`, ''); + return { + collection: collection as unknown as mongo.Collection, + definitionId + }; + } + } + + return null; + } + + private formatBucketDataKey(key: mongo.Document) { + const bucket = (key.b ?? key._id?.b) as string | undefined; + const op = (key.o ?? key._id?.o) as bigint | undefined; + return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; + } + + private tagBucketDataDocument( + document: BucketDataCollectionDocument & { size: number | bigint }, + definitionId: BucketDefinitionId + ): CompactBucketDataDocument { + const tagged = bucketDataDocumentToTagged(document, definitionId); + return { + ...tagged, + size: document.size + }; + } + + private tagClearBucketDataDocument( + document: BucketDataClearProjection, + definitionId: BucketDefinitionId + ): CompactClearBucketDataDocument { + return { + def: definitionId, + _id: { + b: document._id.b, + o: document._id.o + }, + op: document.op, + checksum: document.checksum, + target_op: document.target_op + }; + } + + private collectionBucketDataDocument(document: TaggedBucketDataDocument) { + if (this.db.storageConfig.incrementalReprocessing) { + return taggedBucketDataDocumentToV3(document); + } + return taggedBucketDataDocumentToV1(this.group_id, document); + } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 52e27943a..0afa49b57 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -33,12 +33,15 @@ import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } f import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { VersionedPowerSyncMongo } from './db.js'; import { - BucketDataDocument, - BucketDataKey, + BucketDataDocumentV1, + BucketDataKeyV1, + BucketDataDocumentV3, BucketStateDocument, CommonSourceTableDocument, + LEGACY_BUCKET_DATA_DEFINITION_ID, SourceKey, - StorageConfig + StorageConfig, + bucketDataDocumentToTagged } from './models.js'; import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; @@ -86,7 +89,8 @@ export class MongoSyncBucketStorage this.db = factory.db.versioned(sync_rules.getStorageConfig()); this.checksums = new MongoChecksums(this.db, this.group_id, { ...options.checksumOptions, - storageConfig: options?.storageConfig + storageConfig: options?.storageConfig, + mapping: sync_rules.mapping }); this.writeCheckpointAPI = new MongoWriteCheckpointAPI({ db: this.db, @@ -407,10 +411,15 @@ export class MongoSyncBucketStorage dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions ): AsyncIterable { + if (this.db.storageConfig.incrementalReprocessing) { + yield* this.getBucketDataBatchV3(checkpoint, dataBuckets, options); + return; + } + if (dataBuckets.length == 0) { return; } - let filters: mongo.Filter[] = []; + let filters: mongo.Filter[] = []; const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start])); if (checkpoint == null) { @@ -486,7 +495,10 @@ export class MongoSyncBucketStorage // Ordered by _id, meaning buckets are grouped together for (let rawData of data) { - const row = bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocument; + const row = bucketDataDocumentToTagged( + bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV1, + LEGACY_BUCKET_DATA_DEFINITION_ID + ); const bucket = row._id.b; if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { @@ -552,6 +564,137 @@ export class MongoSyncBucketStorage } } + private async *getBucketDataBatchV3( + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions + ): AsyncIterable { + if (dataBuckets.length == 0) { + return; + } + + if (checkpoint == null) { + throw new ServiceAssertionError('checkpoint is null'); + } + + const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + const end = checkpoint; + let remainingLimit = batchLimit; + let hasMoreAcrossGroups = false; + + const requestsByDefinition = new Map(); + for (const request of dataBuckets) { + const definitionId = this.sync_rules.mapping.bucketSourceId(request.source); + const requests = requestsByDefinition.get(definitionId) ?? []; + requests.push(request); + requestsByDefinition.set(definitionId, requests); + } + + for (const [definitionId, requests] of requestsByDefinition.entries()) { + if (remainingLimit <= 0) { + hasMoreAcrossGroups = true; + break; + } + + const bucketMap = new Map(requests.map((request) => [request.bucket, request.start])); + const filters: mongo.Filter[] = requests.map(({ bucket, start }) => ({ + _id: { + $gt: { + b: bucket, + o: start + }, + $lte: { + b: bucket, + o: end as any + } + } + })); + + const cursor = this.db.bucket_data_v3(this.group_id, definitionId).find( + { + $or: filters + }, + { + session: undefined, + sort: { _id: 1 }, + limit: remainingLimit, + batchSize: remainingLimit + 1, + raw: true, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) as unknown as mongo.FindCursor; + + let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => { + throw lib_mongo.mapQueryError(e, 'while reading bucket data'); + }); + if (data.length == remainingLimit) { + batchHasMore = true; + } + + remainingLimit -= data.length; + hasMoreAcrossGroups ||= batchHasMore; + + let chunkSizeBytes = 0; + let currentChunk: utils.SyncBucketData | null = null; + let targetOp: InternalOpId | null = null; + + for (let rawData of data) { + const row = bucketDataDocumentToTagged( + bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3, + definitionId + ); + const bucket = row._id.b; + + if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { + let start: ProtocolOpId | undefined = undefined; + if (currentChunk != null) { + if (currentChunk.bucket == bucket) { + currentChunk.has_more = true; + start = currentChunk.next_after; + } + + const yieldChunk = currentChunk; + currentChunk = null; + chunkSizeBytes = 0; + yield { chunkData: yieldChunk, targetOp: targetOp }; + targetOp = null; + } + + if (start == null) { + const startOpId = bucketMap.get(bucket); + if (startOpId == null) { + throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`); + } + start = internalToExternalOpId(startOpId); + } + currentChunk = { + bucket, + after: start, + has_more: false, + data: [], + next_after: start + }; + } + + const entry = mapOpEntry(row); + if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) { + targetOp = row.target_op; + } + + currentChunk.data.push(entry); + currentChunk.next_after = entry.op_id; + chunkSizeBytes += rawData.byteLength; + } + + if (currentChunk != null) { + const yieldChunk = currentChunk; + yieldChunk.has_more = batchHasMore || hasMoreAcrossGroups; + yield { chunkData: yieldChunk, targetOp: targetOp }; + } + } + } + async getChecksums( checkpoint: utils.InternalOpId, buckets: storage.BucketChecksumRequest[] @@ -654,12 +797,18 @@ export class MongoSyncBucketStorage }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); - await this.db.bucket_data.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['b', 'o']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); + if (this.db.storageConfig.incrementalReprocessing) { + for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { + await collection.deleteMany({}, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }); + } + } else { + await this.db.bucket_data.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['b', 'o']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ); + } await this.db.bucket_parameters.deleteMany( { 'key.g': this.group_id diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index 5e6a38e90..426b2bc86 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -8,13 +8,14 @@ import { MongoIdSequence } from './MongoIdSequence.js'; import { VersionedPowerSyncMongo } from './db.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { - BucketDataDocument, BucketStateDocument, CommonBucketParameterDocument, CommonCurrentBucket, CommonCurrentLookup, - SourceKey + SourceKey, + TaggedBucketDataDocument } from './models.js'; +import { BucketDefinitionId } from './BucketDefinitionMapping.js'; import { mongoTableId } from '../../utils/util.js'; /** @@ -73,7 +74,7 @@ export interface PersistedBatchOptions { */ export abstract class PersistedBatch { logger: Logger; - bucketData: mongo.AnyBulkWriteOperation[] = []; + bucketData: TaggedBucketDataDocument[] = []; bucketParameters: mongo.AnyBulkWriteOperation[] = []; bucketStates: Map = new Map(); @@ -110,10 +111,16 @@ export abstract class PersistedBatch { protected abstract get currentDataCount(): number; + protected abstract flushBucketData(session: mongo.ClientSession): Promise; + protected abstract flushCurrentData(session: mongo.ClientSession): Promise; protected abstract resetCurrentData(): void; + protected get bucketDataCount(): number { + return this.bucketData.length; + } + protected incrementBucket(bucket: string, op_id: InternalOpId, bytes: number) { let existingState = this.bucketStates.get(bucket); if (existingState) { @@ -129,8 +136,13 @@ export abstract class PersistedBatch { } } + protected flushBucketParameters() { + return this.bucketParameters.length > 0; + } + protected addBucketDataPut(options: { op_id: InternalOpId; + definitionId: BucketDefinitionId; bucket: string; sourceTableId: storage.SourceTable['id']; sourceKey: storage.ReplicaId; @@ -140,27 +152,24 @@ export abstract class PersistedBatch { data: string; }) { this.bucketData.push({ - insertOne: { - document: { - _id: { - g: this.group_id, - b: options.bucket, - o: options.op_id - }, - op: 'PUT', - source_table: mongoTableId(options.sourceTableId), - source_key: options.sourceKey, - table: options.table, - row_id: options.rowId, - checksum: options.checksum, - data: options.data - } - } + def: options.definitionId, + _id: { + b: options.bucket, + o: options.op_id + }, + op: 'PUT', + source_table: mongoTableId(options.sourceTableId), + source_key: options.sourceKey, + table: options.table, + row_id: options.rowId, + checksum: options.checksum, + data: options.data }); } protected addBucketDataRemove(options: { op_id: InternalOpId; + definitionId: BucketDefinitionId; bucket: string; sourceTableId: storage.SourceTable['id']; sourceKey: storage.ReplicaId; @@ -169,33 +178,25 @@ export abstract class PersistedBatch { checksum: bigint; }) { this.bucketData.push({ - insertOne: { - document: { - _id: { - g: this.group_id, - b: options.bucket, - o: options.op_id - }, - op: 'REMOVE', - source_table: mongoTableId(options.sourceTableId), - source_key: options.sourceKey, - table: options.table, - row_id: options.rowId, - checksum: options.checksum, - data: null - } - } + def: options.definitionId, + _id: { + b: options.bucket, + o: options.op_id + }, + op: 'REMOVE', + source_table: mongoTableId(options.sourceTableId), + source_key: options.sourceKey, + table: options.table, + row_id: options.rowId, + checksum: options.checksum, + data: null }); } - protected flushBucketParameters() { - return this.bucketParameters.length > 0; - } - shouldFlushTransaction() { return ( this.currentSize >= MAX_TRANSACTION_BATCH_SIZE || - this.bucketData.length >= MAX_TRANSACTION_DOC_COUNT || + this.bucketDataCount >= MAX_TRANSACTION_DOC_COUNT || this.currentDataCount >= MAX_TRANSACTION_DOC_COUNT || this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT ); @@ -205,12 +206,9 @@ export abstract class PersistedBatch { const db = this.db; const startAt = performance.now(); let flushedSomething = false; - if (this.bucketData.length > 0) { + if (this.bucketDataCount > 0) { flushedSomething = true; - await db.bucket_data.bulkWrite(this.bucketData, { - session, - ordered: false - }); + await this.flushBucketData(session); } if (this.flushBucketParameters()) { flushedSomething = true; @@ -238,14 +236,14 @@ export abstract class PersistedBatch { const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000); this.logger.info( - `Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${ + `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${ this.currentDataCount } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`, { flushed: { duration: duration, size: this.currentSize, - bucket_data_count: this.bucketData.length, + bucket_data_count: this.bucketDataCount, parameter_data_count: this.bucketParameters.length, current_data_count: this.currentDataCount, replication_lag_seconds: replicationLag @@ -254,14 +252,14 @@ export abstract class PersistedBatch { ); } else { this.logger.info( - `Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${ + `Flushed ${this.bucketDataCount} + ${this.bucketParameters.length} + ${ this.currentDataCount } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`, { flushed: { duration: duration, size: this.currentSize, - bucket_data_count: this.bucketData.length, + bucket_data_count: this.bucketDataCount, parameter_data_count: this.bucketParameters.length, current_data_count: this.currentDataCount } @@ -271,7 +269,7 @@ export abstract class PersistedBatch { } const stats = { - bucketDataCount: this.bucketData.length, + bucketDataCount: this.bucketDataCount, parameterDataCount: this.bucketParameters.length, currentDataCount: this.currentDataCount, flushedAny: flushedSomething diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index 5e505e723..d3cfdd1ad 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -15,7 +15,9 @@ import { BucketParameterDocument, CurrentBucket, CurrentDataDocument, + LEGACY_BUCKET_DATA_DEFINITION_ID, SourceKey, + taggedBucketDataDocumentToV1, isCurrentBucketV3, isRecordedLookupV3 } from './models.js'; @@ -58,6 +60,7 @@ export class PersistedBatchV1 extends PersistedBatch { this.addBucketDataPut({ op_id, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, bucket: evaluated.bucket, sourceTableId: options.table.id, sourceKey: options.sourceKey, @@ -75,6 +78,7 @@ export class PersistedBatchV1 extends PersistedBatch { this.addBucketDataRemove({ op_id, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, bucket: bucket.bucket, sourceTableId: options.table.id, sourceKey: options.sourceKey, @@ -194,6 +198,20 @@ export class PersistedBatchV1 extends PersistedBatch { return this.currentData.length; } + protected async flushBucketData(session: mongo.ClientSession) { + await this.db.v1_bucket_data.bulkWrite( + this.bucketData.map((document) => ({ + insertOne: { + document: taggedBucketDataDocumentToV1(this.group_id, document) + } + })), + { + session, + ordered: false + } + ); + } + protected async flushCurrentData(session: mongo.ClientSession) { await this.db.v1_current_data.bulkWrite(this.currentData, { session, diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 8fb17aa7b..ba9b2d4e1 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -4,6 +4,7 @@ import { storage, utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; +import { BucketDefinitionId } from './BucketDefinitionMapping.js'; import { PersistedBatch, SaveBucketDataOptions, @@ -17,7 +18,8 @@ import { isCurrentBucketV3, isRecordedLookupV3, RecordedLookupV3, - SourceKey + SourceKey, + taggedBucketDataDocumentToV3 } from './models.js'; export class PersistedBatchV3 extends PersistedBatch { @@ -59,6 +61,7 @@ export class PersistedBatchV3 extends PersistedBatch { this.addBucketDataPut({ op_id, + definitionId: sourceDefinitionId, bucket: evaluated.bucket, sourceTableId: options.table.id, sourceKey: options.sourceKey, @@ -76,6 +79,7 @@ export class PersistedBatchV3 extends PersistedBatch { this.addBucketDataRemove({ op_id, + definitionId: bucket.def, bucket: bucket.bucket, sourceTableId: options.table.id, sourceKey: options.sourceKey, @@ -212,6 +216,29 @@ export class PersistedBatchV3 extends PersistedBatch { return this.currentData.length; } + protected async flushBucketData(session: mongo.ClientSession) { + const operationsByDefinition = new Map(); + for (const document of this.bucketData) { + const existing = operationsByDefinition.get(document.def) ?? []; + existing.push(document); + operationsByDefinition.set(document.def, existing); + } + + for (const [definitionId, documents] of operationsByDefinition.entries()) { + await this.db.bucket_data_v3(this.group_id, definitionId).bulkWrite( + documents.map((document) => ({ + insertOne: { + document: taggedBucketDataDocumentToV3(document) + } + })), + { + session, + ordered: false + } + ); + } + } + protected async flushCurrentData(session: mongo.ClientSession) { await this.db.v3_current_data.bulkWrite(this.currentData, { session, diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 5bc0f8281..1724f9cba 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -4,7 +4,8 @@ import { POWERSYNC_VERSION, storage } from '@powersync/service-core'; import { MongoStorageConfig } from '../../types/types.js'; import { - BucketDataDocument, + BucketDataDocumentV1, + BucketDataDocumentV3, BucketParameterDocument, BucketParameterDocumentV3, BucketStateDocument, @@ -25,6 +26,7 @@ import { WriteCheckpointDocument } from './models.js'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { BucketDefinitionId } from './BucketDefinitionMapping.js'; export interface PowerSyncMongoOptions { /** @@ -36,7 +38,7 @@ export interface PowerSyncMongoOptions { export class PowerSyncMongo { readonly current_data: mongo.Collection; readonly v3_current_data: mongo.Collection; - readonly bucket_data: mongo.Collection; + readonly bucket_data: mongo.Collection; readonly bucket_parameters: mongo.Collection; readonly v3_bucket_parameters: mongo.Collection; readonly op_id_sequence: mongo.Collection; @@ -84,6 +86,23 @@ export class PowerSyncMongo { return new VersionedPowerSyncMongo(this, storageConfig); } + bucketDataCollectionNameV3(groupId: number, definitionId: BucketDefinitionId) { + return `bucket_data_${groupId}_${definitionId}`; + } + + bucketDataV3(groupId: number, definitionId: BucketDefinitionId): mongo.Collection { + return this.db.collection(this.bucketDataCollectionNameV3(groupId, definitionId)); + } + + async listBucketDataCollectionsV3(groupId?: number): Promise[]> { + const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`; + const collections = await this.db.listCollections({}, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); + } + /** * Clear all collections. */ @@ -91,6 +110,9 @@ export class PowerSyncMongo { await this.current_data.deleteMany({}); await this.v3_current_data.deleteMany({}); await this.bucket_data.deleteMany({}); + for (const collection of await this.listBucketDataCollectionsV3()) { + await collection.deleteMany({}); + } await this.bucket_parameters.deleteMany({}); await this.v3_bucket_parameters.deleteMany({}); await this.op_id_sequence.deleteMany({}); @@ -285,6 +307,33 @@ export class VersionedPowerSyncMongo { return this.#upstream.bucket_data; } + get v1_bucket_data() { + if (this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'bucket_data collection should not be used when incrementalReprocessing is enabled' + ); + } + return this.#upstream.bucket_data; + } + + bucket_data_v3(groupId: number, definitionId: BucketDefinitionId) { + if (!this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'v3 bucket_data collections should not be used when incrementalReprocessing is disabled' + ); + } + return this.#upstream.bucketDataV3(groupId, definitionId); + } + + listBucketDataCollectionsV3(groupId?: number) { + if (!this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'v3 bucket_data collections should not be used when incrementalReprocessing is disabled' + ); + } + return this.#upstream.listBucketDataCollectionsV3(groupId); + } + get bucket_parameters() { if (this.storageConfig.incrementalReprocessing) { return this.#upstream.v3_bucket_parameters as unknown as mongo.Collection; diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 1b05d836e..900c5b4e1 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -23,7 +23,7 @@ export interface SourceKey { k: ReplicaId; } -export interface BucketDataKey { +export interface BucketDataKeyV1 { /** group_id */ g: number; /** bucket name */ @@ -32,6 +32,13 @@ export interface BucketDataKey { o: bigint; } +export interface BucketDataKeyV3 { + /** bucket name */ + b: string; + /** op_id */ + o: bigint; +} + export interface CurrentDataDocument { _id: SourceKey; data: bson.Binary; @@ -78,8 +85,7 @@ export interface BucketParameterDocumentV3 extends BucketParameterDocument { index: ParameterIndexId; } -export interface BucketDataDocument { - _id: BucketDataKey; +export interface BucketDataProperties { op: OpType; source_table?: bson.ObjectId; source_key?: ReplicaId; @@ -90,6 +96,61 @@ export interface BucketDataDocument { target_op?: bigint | null; } +export interface BucketDataDocumentV1 extends BucketDataProperties { + _id: BucketDataKeyV1; +} + +export interface BucketDataDocumentV3 extends BucketDataProperties { + _id: BucketDataKeyV3; +} + +/** + * All data we need in-memory, for any storage version. + */ +export interface TaggedBucketDataDocument extends BucketDataProperties { + def: BucketDefinitionId; + _id: BucketDataKeyV3; +} + +/** + * Internal-only tag used for v1 bucket_data rows before they are converted to the v1 on-disk shape. + */ +export const LEGACY_BUCKET_DATA_DEFINITION_ID = '0'; + +export function bucketDataDocumentToTagged( + document: BucketDataDocumentV1 | BucketDataDocumentV3, + definitionId: BucketDefinitionId +): TaggedBucketDataDocument { + return { + ...document, + def: definitionId, + _id: { + b: document._id.b, + o: document._id.o + } + }; +} + +export function taggedBucketDataDocumentToV1( + groupId: number, + document: TaggedBucketDataDocument +): BucketDataDocumentV1 { + const { def: _definitionId, _id: _id, ...rest } = document; + return { + _id: { + g: groupId, + b: _id.b, + o: _id.o + }, + ...rest + }; +} + +export function taggedBucketDataDocumentToV3(document: TaggedBucketDataDocument): BucketDataDocumentV3 { + const { def: _definitionId, ...rest } = document; + return rest; +} + export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; export interface SourceTableDocument { @@ -106,8 +167,8 @@ export interface SourceTableDocument { } export interface SourceTableDocumentV3 extends SourceTableDocument { - bucket_data_source_ids: number[]; - parameter_lookup_source_ids: number[]; + bucket_data_source_ids: BucketDefinitionId[]; + parameter_lookup_source_ids: ParameterIndexId[]; } export interface SourceTableDocumentSnapshotStatus { @@ -330,5 +391,5 @@ export function isCurrentBucketV3(bucket: CommonCurrentBucket): bucket is Curren } export function isRecordedLookupV3(lookup: CommonCurrentLookup): lookup is RecordedLookupV3 { - return typeof lookup === 'object' && lookup != null && 'd' in lookup && 'l' in lookup; + return typeof lookup === 'object' && lookup != null && 'i' in lookup && 'l' in lookup; } diff --git a/modules/module-mongodb-storage/src/utils/util.ts b/modules/module-mongodb-storage/src/utils/util.ts index 40d5e0934..06342cbb9 100644 --- a/modules/module-mongodb-storage/src/utils/util.ts +++ b/modules/module-mongodb-storage/src/utils/util.ts @@ -4,7 +4,7 @@ import * as uuid from 'uuid'; import { mongo } from '@powersync/lib-service-mongodb'; import { storage, utils } from '@powersync/service-core'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { BucketDataDocument } from '../storage/implementation/models.js'; +import { TaggedBucketDataDocument } from '../storage/implementation/models.js'; export function idPrefixFilter(prefix: Partial, rest: (keyof T)[]): mongo.Condition { let filter = { @@ -69,7 +69,7 @@ export async function readSingleBatch(cursor: mongo.AbstractCursor): Promi } } -export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry { +export function mapOpEntry(row: TaggedBucketDataDocument): utils.OplogEntry { if (row.op == 'PUT' || row.op == 'REMOVE') { return { op_id: utils.internalToExternalOpId(row._id.o), diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index e749f5cf2..558a1f0c0 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -129,9 +129,25 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor // Test that the checksum type is correct. // Specifically, test that it never persisted as double. const mongoFactory = factory as MongoBucketStorage; - const checksumTypes = await mongoFactory.db.bucket_data - .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }]) - .toArray(); + const checksumTypes = + storageVersion >= 3 + ? ( + await Promise.all( + ( + await mongoFactory.db.db + .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true }) + .toArray() + ).map((collection: { name: string }) => + mongoFactory.db.db + .collection(collection.name) + .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }]) + .toArray() + ) + ) + ).flat() + : await mongoFactory.db.bucket_data + .aggregate([{ $group: { _id: { $type: '$checksum' }, count: { $sum: 1 } } }]) + .toArray(); expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]); }); @@ -166,7 +182,14 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const mongoFactory = factory as MongoBucketStorage; const currentData = await mongoFactory.db.v3_current_data.findOne({}); const firstBucket: CurrentDataDocumentV3['buckets'][number] | undefined = currentData?.buckets[0]; - expect(firstBucket?.def).toBeGreaterThan(0); + expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/); + + const bucketCollections = await mongoFactory.db.db + .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true }) + .toArray(); + expect( + bucketCollections.some((collection) => collection.name === `bucket_data_${syncRules.id}_${firstBucket?.def}`) + ).toBe(true); const syncRule = await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id }); const ruleMapping: SyncRuleDocument['rule_mapping'] | undefined = syncRule?.rule_mapping; From 37fef1528955dc7a0ee63d8209de4fccc5986b0e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 17:07:10 +0200 Subject: [PATCH 11/93] Use clustered collections. --- .../implementation/BucketDefinitionMapping.ts | 4 ++++ .../implementation/MongoSyncBucketStorage.ts | 16 ++++++++++++++++ .../src/storage/implementation/db.ts | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts index 4656b06be..586a622e4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -42,6 +42,10 @@ export class BucketDefinitionMapping { return defId; } + allBucketDefinitionIds(): BucketDefinitionId[] { + return Object.values(this.definitions); + } + parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId { const key = `${source.defaultLookupScope.lookupName}#${source.defaultLookupScope.queryId}`; const defId = this.parameterLookupMapping[key]; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 0afa49b57..710cbaf2f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -76,6 +76,7 @@ export class MongoSyncBucketStorage private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined; private writeCheckpointAPI: MongoWriteCheckpointAPI; + #storageInitialized = false; constructor( public readonly factory: MongoBucketStorage, @@ -172,7 +173,22 @@ export class MongoSyncBucketStorage }); } + private async initializeStorage() { + if (this.#storageInitialized) { + return; + } + + const mapping = this.sync_rules.mapping; + for (let source of mapping.allBucketDefinitionIds()) { + const collection = this.db.bucket_data_v3(this.group_id, source).collectionName; + await this.db.db.createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }); + } + this.#storageInitialized = true; + } + async createWriter(options: storage.CreateWriterOptions): Promise { + await this.initializeStorage(); + const doc = await this.db.sync_rules.findOne( { _id: this.group_id diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 1724f9cba..982fd7e32 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -26,7 +26,7 @@ import { WriteCheckpointDocument } from './models.js'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { BucketDefinitionId } from './BucketDefinitionMapping.js'; +import { BucketDefinitionId, BucketDefinitionMapping } from './BucketDefinitionMapping.js'; export interface PowerSyncMongoOptions { /** From a4226c1f03d7ecc626d81439955d67bb8d804ffe Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 17:08:06 +0200 Subject: [PATCH 12/93] Drop bucket_data collections when clearing. --- .../src/storage/implementation/MongoSyncBucketStorage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 710cbaf2f..54bef6775 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -815,7 +815,7 @@ export class MongoSyncBucketStorage ); if (this.db.storageConfig.incrementalReprocessing) { for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { - await collection.deleteMany({}, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }); + await collection.drop(); } } else { await this.db.bucket_data.deleteMany( From 5db7eba76208542d71a68e5411fac98277f7f604 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 17:12:06 +0200 Subject: [PATCH 13/93] Fix type issues. --- .../src/storage/MongoBucketStorage.ts | 2 +- .../src/storage/implementation/MongoChecksums.ts | 9 +++++++-- .../src/storage/implementation/MongoSyncBucketStorage.ts | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 78830c549..90fd56f16 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -16,7 +16,7 @@ import { MongoChecksumOptions } from './implementation/MongoChecksums.js'; import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; export interface MongoBucketStorageOptions { - checksumOptions?: Omit; + checksumOptions?: Omit; } export class MongoBucketStorage extends storage.BucketStorageFactory { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index 2d9cc73ea..77b534570 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -1,5 +1,6 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; +import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { addPartialChecksums, bson, @@ -33,7 +34,7 @@ export interface MongoChecksumOptions { operationBatchLimit?: number; storageConfig: StorageConfig; - mapping: BucketDefinitionMapping; + mapping?: BucketDefinitionMapping; } const DEFAULT_BUCKET_BATCH_LIMIT = 200; @@ -51,7 +52,7 @@ const DEFAULT_OPERATION_BATCH_LIMIT = 50_000; export class MongoChecksums { private _cache: ChecksumCache | undefined; private readonly storageConfig: StorageConfig; - private readonly mapping: BucketDefinitionMapping; + private readonly mapping: BucketDefinitionMapping | undefined; constructor( private db: VersionedPowerSyncMongo, @@ -206,6 +207,10 @@ export class MongoChecksums { */ private async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { if (this.storageConfig.incrementalReprocessing) { + if (this.mapping == null) { + throw new ServiceAssertionError('BucketDefinitionMapping is required for v3 MongoDB checksum queries'); + } + const results = new Map(); const requestsByDefinition = new Map(); const fallbackRequests: FetchPartialBucketChecksum[] = []; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 54bef6775..4bfe2355a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -52,7 +52,7 @@ import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; export interface MongoSyncBucketStorageOptions { - checksumOptions?: Omit; + checksumOptions?: Omit; storageConfig: StorageConfig; } From e45066e4f09622958dddc8487cac0f8672780219 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 17:28:47 +0200 Subject: [PATCH 14/93] Fixes. --- .../implementation/MongoSyncBucketStorage.ts | 37 ++++++++++++++----- .../src/storage/implementation/db.ts | 2 +- .../__snapshots__/storage_sync.test.ts.snap | 2 + 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 4bfe2355a..38fe1f048 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -580,6 +580,17 @@ export class MongoSyncBucketStorage } } + /** + * Reads V3 bucket data across per-definition collections while presenting a single paginated + * stream to the caller. + * + * Unlike v1, the requested buckets may live in multiple collections. We therefore page through + * one definition group at a time. + * + * Important: as soon as any limit is hit for the current read, we stop and return control to + * the caller. We do not continue with the same group, and we do not move on to later groups. + * That keeps pagination boundaries predictable and matches the v1 behavior more closely. + */ private async *getBucketDataBatchV3( checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], @@ -597,7 +608,6 @@ export class MongoSyncBucketStorage const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; const end = checkpoint; let remainingLimit = batchLimit; - let hasMoreAcrossGroups = false; const requestsByDefinition = new Map(); for (const request of dataBuckets) { @@ -607,14 +617,12 @@ export class MongoSyncBucketStorage requestsByDefinition.set(definitionId, requests); } - for (const [definitionId, requests] of requestsByDefinition.entries()) { - if (remainingLimit <= 0) { - hasMoreAcrossGroups = true; - break; - } - + const definitionGroups = Array.from(requestsByDefinition.entries()); + for (let groupIndex = 0; groupIndex < definitionGroups.length && remainingLimit > 0; groupIndex++) { + const [definitionId, requests] = definitionGroups[groupIndex]; + const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1; const bucketMap = new Map(requests.map((request) => [request.bucket, request.start])); - const filters: mongo.Filter[] = requests.map(({ bucket, start }) => ({ + const filters: mongo.Filter[] = Array.from(bucketMap.entries()).map(([bucket, start]) => ({ _id: { $gt: { b: bucket, @@ -647,9 +655,11 @@ export class MongoSyncBucketStorage if (data.length == remainingLimit) { batchHasMore = true; } + if (data.length == 0) { + continue; + } remainingLimit -= data.length; - hasMoreAcrossGroups ||= batchHasMore; let chunkSizeBytes = 0; let currentChunk: utils.SyncBucketData | null = null; @@ -705,9 +715,16 @@ export class MongoSyncBucketStorage if (currentChunk != null) { const yieldChunk = currentChunk; - yieldChunk.has_more = batchHasMore || hasMoreAcrossGroups; + // Stop after the current read if either: + // 1. MongoDB indicates more rows remain for this definition group, or + // 2. we exhausted the caller's overall document limit before later groups. + yieldChunk.has_more = batchHasMore || (remainingLimit <= 0 && hasLaterDefinitionGroups); yield { chunkData: yieldChunk, targetOp: targetOp }; } + + if (batchHasMore || remainingLimit <= 0) { + return; + } } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 982fd7e32..a3498c180 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -111,7 +111,7 @@ export class PowerSyncMongo { await this.v3_current_data.deleteMany({}); await this.bucket_data.deleteMany({}); for (const collection of await this.listBucketDataCollectionsV3()) { - await collection.deleteMany({}); + await collection.drop(); } await this.bucket_parameters.deleteMany({}); await this.v3_bucket_parameters.deleteMany({}); diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap index d59bfb500..70387ff0e 100644 --- a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -3284,3 +3284,5 @@ exports[`sync - mongodb > storage v3 > sync updates to parameter query only 2`] }, ] `; + +exports[`sync - mongodb > storage v3 > write checkpoint 1`] = `[]`; From 9a02f5ff267c5d92ed19384c7a4ad2075cd16565 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 17:32:36 +0200 Subject: [PATCH 15/93] Fix tests. --- .../test/src/slow_tests.test.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index 77ce26cf2..566597f9c 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -185,10 +185,22 @@ bucket_definitions: if (f instanceof mongo_storage.storage.MongoBucketStorage) { const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) .filter((row) => row._id.o <= checkpoint) + .map((row) => + mongo_storage.storage.bucketDataDocumentToTagged( + row, + mongo_storage.storage.LEGACY_BUCKET_DATA_DEFINITION_ID + ) + ) .map(mongo_storage.storage.mapOpEntry); await storage.compact({ maxOpId: checkpoint }); const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) .filter((row) => row._id.o <= checkpoint) + .map((row) => + mongo_storage.storage.bucketDataDocumentToTagged( + row, + mongo_storage.storage.LEGACY_BUCKET_DATA_DEFINITION_ID + ) + ) .map(mongo_storage.storage.mapOpEntry); test_utils.validateCompactedBucket(opsBefore, opsAfter); @@ -252,7 +264,11 @@ bucket_definitions: const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray(); // All a single bucket in this test - const bucket = ops.map((op) => mongo_storage.storage.mapOpEntry(op)); + const bucket = ops + .map((op) => + mongo_storage.storage.bucketDataDocumentToTagged(op, mongo_storage.storage.LEGACY_BUCKET_DATA_DEFINITION_ID) + ) + .map((op) => mongo_storage.storage.mapOpEntry(op)); const reduced = test_utils.reduceBucket(bucket); expect(reduced).toMatchObject([ { From aba9997d053e84ac7cd5acb84fbf16b786729990 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 17:39:57 +0200 Subject: [PATCH 16/93] Split out checksum implementations. --- .../storage/implementation/MongoChecksums.ts | 247 ++++++++++-------- 1 file changed, 142 insertions(+), 105 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index 77b534570..d8c0222b4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -41,7 +41,7 @@ const DEFAULT_BUCKET_BATCH_LIMIT = 200; const DEFAULT_OPERATION_BATCH_LIMIT = 50_000; /** - * Checksum query implementation. + * Shared checksum query plumbing. * * General implementation flow is: * 1. getChecksums() -> check cache for (partial) matches. If not found or partial match, query the remainder using computePartialChecksums(). @@ -49,18 +49,16 @@ const DEFAULT_OPERATION_BATCH_LIMIT = 50_000; * 3. computePartialChecksumsDirect() -> split into batches of 200 buckets at a time -> computePartialChecksumsInternal() * 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time */ -export class MongoChecksums { +abstract class AbstractMongoChecksums { private _cache: ChecksumCache | undefined; private readonly storageConfig: StorageConfig; - private readonly mapping: BucketDefinitionMapping | undefined; constructor( - private db: VersionedPowerSyncMongo, - private group_id: number, - private options: MongoChecksumOptions + protected readonly db: VersionedPowerSyncMongo, + protected readonly group_id: number, + protected readonly options: MongoChecksumOptions ) { this.storageConfig = options.storageConfig; - this.mapping = options.mapping; } /** @@ -205,105 +203,9 @@ export class MongoChecksums { * * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this. */ - private async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { - if (this.storageConfig.incrementalReprocessing) { - if (this.mapping == null) { - throw new ServiceAssertionError('BucketDefinitionMapping is required for v3 MongoDB checksum queries'); - } - - const results = new Map(); - const requestsByDefinition = new Map(); - const fallbackRequests: FetchPartialBucketChecksum[] = []; - for (const request of batch) { - if (!isBucketSourceLike(request.source)) { - fallbackRequests.push(request); - continue; - } - const definitionId = this.mapping.bucketSourceId(request.source); - const existing = requestsByDefinition.get(definitionId) ?? []; - existing.push(request); - requestsByDefinition.set(definitionId, existing); - } - - for (const [definitionId, requests] of requestsByDefinition.entries()) { - const groupResults = await this.computePartialChecksumsForCollection( - requests, - this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, - (request) => ({ - _id: { - $gt: { - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - b: request.bucket, - o: request.end - } - } - }) - ); - for (const checksum of groupResults.values()) { - results.set(checksum.bucket, checksum); - } - } - - if (fallbackRequests.length > 0) { - const collections = await this.db.listBucketDataCollectionsV3(this.group_id); - for (const request of fallbackRequests) { - let merged: PartialOrFullChecksum | null = null; - for (const collection of collections) { - const groupResults = await this.computePartialChecksumsForCollection( - [request], - collection as unknown as mongo.Collection, - (entry) => ({ - _id: { - $gt: { - b: entry.bucket, - o: entry.start ?? new bson.MinKey() - }, - $lte: { - b: entry.bucket, - o: entry.end - } - } - }) - ); - merged = addPartialChecksums(request.bucket, merged, groupResults.get(request.bucket) ?? null); - } - results.set( - request.bucket, - merged ?? - (request.start == null - ? { bucket: request.bucket, count: 0, checksum: 0 } - : { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }) - ); - } - } - - return results; - } - - return this.computePartialChecksumsForCollection( - batch, - this.db.bucket_data as unknown as mongo.Collection, - (request) => ({ - _id: { - $gt: { - g: this.group_id, - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - g: this.group_id, - b: request.bucket, - o: request.end - } - } - }) - ); - } + protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise; - private async computePartialChecksumsForCollection( + protected async computePartialChecksumsForCollection( batch: FetchPartialBucketChecksum[], collection: mongo.Collection, createFilter: (request: FetchPartialBucketChecksum) => any @@ -431,6 +333,141 @@ export class MongoChecksums { } } +class MongoChecksumsV1Impl extends AbstractMongoChecksums { + protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + return this.computePartialChecksumsForCollection( + batch, + this.db.bucket_data as unknown as mongo.Collection, + (request) => ({ + _id: { + $gt: { + g: this.group_id, + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + g: this.group_id, + b: request.bucket, + o: request.end + } + } + }) + ); + } +} + +class MongoChecksumsV3Impl extends AbstractMongoChecksums { + constructor( + db: VersionedPowerSyncMongo, + group_id: number, + options: MongoChecksumOptions, + private readonly mapping: BucketDefinitionMapping + ) { + super(db, group_id, options); + } + + protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + const results = new Map(); + const requestsByDefinition = new Map(); + const fallbackRequests: FetchPartialBucketChecksum[] = []; + + for (const request of batch) { + if (!isBucketSourceLike(request.source)) { + fallbackRequests.push(request); + continue; + } + + const definitionId = this.mapping.bucketSourceId(request.source); + const existing = requestsByDefinition.get(definitionId) ?? []; + existing.push(request); + requestsByDefinition.set(definitionId, existing); + } + + for (const [definitionId, requests] of requestsByDefinition.entries()) { + const groupResults = await this.computePartialChecksumsForCollection( + requests, + this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, + createV3BucketFilter + ); + for (const checksum of groupResults.values()) { + results.set(checksum.bucket, checksum); + } + } + + if (fallbackRequests.length > 0) { + const collections = await this.db.listBucketDataCollectionsV3(this.group_id); + for (const request of fallbackRequests) { + let merged: PartialOrFullChecksum | null = null; + for (const collection of collections) { + const groupResults = await this.computePartialChecksumsForCollection( + [request], + collection as unknown as mongo.Collection, + createV3BucketFilter + ); + merged = addPartialChecksums(request.bucket, merged, groupResults.get(request.bucket) ?? null); + } + results.set(request.bucket, merged ?? emptyChecksumForRequest(request)); + } + } + + return results; + } +} + +/** + * Public checksum API. Delegates to a storage-version-specific implementation. + */ +export class MongoChecksums { + private readonly impl: AbstractMongoChecksums; + + constructor(db: VersionedPowerSyncMongo, group_id: number, options: MongoChecksumOptions) { + this.impl = options.storageConfig.incrementalReprocessing + ? new MongoChecksumsV3Impl( + db, + group_id, + options, + options.mapping ?? + (() => { + throw new ServiceAssertionError('BucketDefinitionMapping is required for v3 MongoDB checksum queries'); + })() + ) + : new MongoChecksumsV1Impl(db, group_id, options); + } + + async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise { + return this.impl.getChecksums(checkpoint, buckets); + } + + clearCache() { + this.impl.clearCache(); + } + + async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise { + return this.impl.computePartialChecksumsDirect(batch); + } +} + +function createV3BucketFilter(request: FetchPartialBucketChecksum) { + return { + _id: { + $gt: { + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + b: request.bucket, + o: request.end + } + } + }; +} + +function emptyChecksumForRequest(request: FetchPartialBucketChecksum): PartialOrFullChecksum { + return request.start == null + ? { bucket: request.bucket, count: 0, checksum: 0 } + : { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }; +} + function isBucketSourceLike( source: FetchPartialBucketChecksum['source'] ): source is NonNullable { From 0477215675353da4d7d1ec603382ece56dccf9ef Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 17:52:29 +0200 Subject: [PATCH 17/93] Split bucket_parameter collections. --- .../src/storage/MongoBucketStorage.ts | 18 ++- .../implementation/MongoParameterCompactor.ts | 30 +++- .../implementation/MongoSyncBucketStorage.ts | 140 +++++++++++++++++- .../storage/implementation/PersistedBatch.ts | 17 +-- .../implementation/PersistedBatchV1.ts | 26 +++- .../implementation/PersistedBatchV3.ts | 37 ++++- .../src/storage/implementation/db.ts | 84 ++++++++--- .../src/storage/implementation/models.ts | 30 +++- 8 files changed, 321 insertions(+), 61 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 90fd56f16..413e616e9 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -346,6 +346,20 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ]) .toArray() .catch(ignoreNotExisting); + const v3_parameter_aggregates = await Promise.all( + (await this.db.listBucketParameterCollectionsV3()).map((collection) => + collection + .aggregate([ + { + $collStats: { + storageStats: {} + } + } + ]) + .toArray() + .catch(ignoreNotExisting) + ) + ); const v1_replication_aggregate = await this.db.current_data .aggregate([ @@ -372,7 +386,9 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { operations_size_bytes: Number(operations_aggregate[0].storageStats.size) + v3_operation_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0), - parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size), + parameters_size_bytes: + Number(parameters_aggregate[0].storageStats.size) + + v3_parameter_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0), replication_size_bytes: Number(v1_replication_aggregate[0].storageStats.size) + Number(v3_replication_aggregate[0].storageStats.size) }; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts index 0fd7024f9..ce3a2cf72 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts @@ -3,7 +3,7 @@ import { logger } from '@powersync/lib-services-framework'; import { bson, CompactOptions, InternalOpId } from '@powersync/service-core'; import { LRUCache } from 'lru-cache'; import { VersionedPowerSyncMongo } from './db.js'; -import { BucketParameterDocument } from './models.js'; +import { BucketParameterDocument, BucketParameterDocumentV3 } from './models.js'; /** * Compacts parameter lookup data (the bucket_parameters collection). @@ -22,6 +22,24 @@ export class MongoParameterCompactor { async compact() { logger.info(`Compacting parameters for group ${this.group_id} up to checkpoint ${this.checkpoint}`); + if (this.db.storageConfig.incrementalReprocessing) { + await this.compactV3(); + return; + } + await this.compactV1(); + } + + private async compactV1() { + await this.compactCollection(this.db.v1_bucket_parameters); + } + + private async compactV3() { + for (const collection of await this.db.listBucketParameterCollectionsV3(this.group_id)) { + await this.compactCollection(collection); + } + } + + private async compactCollection(collection: mongo.Collection) { // This is the currently-active checkpoint. // We do not remove any data that may be used by this checkpoint. // snapshot queries ensure that if any clients are still using older checkpoints, they would @@ -32,7 +50,7 @@ export class MongoParameterCompactor { // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup) // in MongoDB already. However, that risks running into cases where MongoDB needs to process // very large amounts of data before returning results, which could lead to timeouts. - const cursor = this.db.bucket_parameters.find( + const cursor = collection.find( { 'key.g': this.group_id }, @@ -48,17 +66,17 @@ export class MongoParameterCompactor { max: this.options.compactParameterCacheLimit ?? 10_000 }); let removeIds: InternalOpId[] = []; - let removeDeleted: mongo.AnyBulkWriteOperation[] = []; + let removeDeleted: mongo.AnyBulkWriteOperation[] = []; const flush = async (force: boolean) => { if (removeIds.length >= 1000 || (force && removeIds.length > 0)) { - const results = await this.db.bucket_parameters.deleteMany({ _id: { $in: removeIds } }); + const results = await collection.deleteMany({ _id: { $in: removeIds } }); logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`); removeIds = []; } if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) { - const results = await this.db.bucket_parameters.bulkWrite(removeDeleted); + const results = await collection.bulkWrite(removeDeleted); logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`); removeDeleted = []; } @@ -100,6 +118,6 @@ export class MongoParameterCompactor { } await flush(true); - logger.info('Parameter compaction completed'); + logger.info(`Parameter compaction completed for ${collection.collectionName}`); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 38fe1f048..ed0f4ac0a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -359,6 +359,16 @@ export class MongoSyncBucketStorage async getParameterSets( checkpoint: MongoReplicationCheckpoint, lookups: ScopedParameterLookup[] + ): Promise { + if (this.db.storageConfig.incrementalReprocessing) { + return this.getParameterSetsV3(checkpoint, lookups); + } + return this.getParameterSetsV1(checkpoint, lookups); + } + + private async getParameterSetsV1( + checkpoint: MongoReplicationCheckpoint, + lookups: ScopedParameterLookup[] ): Promise { return this.db.client.withSession({ snapshot: true }, async (session) => { // Set the session's snapshot time to the checkpoint's snapshot time. @@ -380,7 +390,7 @@ export class MongoSyncBucketStorage // but could not do the same using $group. // For now, just rely on compacting to remove extraneous data. // For a description of the data format, see the `/docs/parameters-lookups.md` file. - const rows = await this.db.bucket_parameters + const rows = await this.db.v1_bucket_parameters .aggregate( [ { @@ -422,6 +432,65 @@ export class MongoSyncBucketStorage }); } + private async getParameterSetsV3( + checkpoint: MongoReplicationCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise { + return this.db.client.withSession({ snapshot: true }, async (session) => { + setSessionSnapshotTime(session, checkpoint.snapshotTime); + + const lookupsByIndex = new Map(); + for (const lookup of lookups) { + const indexId = this.sync_rules.mapping.parameterLookupId(lookup.source); + const existing = lookupsByIndex.get(indexId) ?? []; + existing.push(storage.serializeLookup(lookup)); + lookupsByIndex.set(indexId, existing); + } + + const groupedParameters: SqliteJsonRow[][] = []; + for (const [indexId, lookupFilter] of lookupsByIndex.entries()) { + const rows = await this.db + .bucket_parameters_v3(this.group_id, indexId) + .aggregate( + [ + { + $match: { + lookup: { $in: lookupFilter }, + _id: { $lte: checkpoint.checkpoint } + } + }, + { + $sort: { + _id: -1 + } + }, + { + $group: { + _id: { key: '$key', lookup: '$lookup' }, + bucket_parameters: { + $first: '$bucket_parameters' + } + } + } + ], + { + session, + readConcern: 'snapshot', + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) + .toArray() + .catch((e) => { + throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); + }); + + groupedParameters.push(...rows.map((row) => row.bucket_parameters)); + } + + return groupedParameters.flat(); + }); + } + async *getBucketDataBatch( checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], @@ -842,12 +911,18 @@ export class MongoSyncBucketStorage { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); } - await this.db.bucket_parameters.deleteMany( - { - 'key.g': this.group_id - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); + if (this.db.storageConfig.incrementalReprocessing) { + for (const collection of await this.db.listBucketParameterCollectionsV3(this.group_id)) { + await collection.drop(); + } + } else { + await this.db.v1_bucket_parameters.deleteMany( + { + 'key.g': this.group_id + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ); + } await this.db.common_current_data.deleteMany( { @@ -1169,9 +1244,18 @@ export class MongoSyncBucketStorage private async getParameterBucketChanges( options: GetCheckpointChangesOptions + ): Promise> { + if (this.db.storageConfig.incrementalReprocessing) { + return this.getParameterBucketChangesV3(options); + } + return this.getParameterBucketChangesV1(options); + } + + private async getParameterBucketChangesV1( + options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; - const parameterUpdates = await this.db.bucket_parameters + const parameterUpdates = await this.db.v1_bucket_parameters .find( { _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint }, @@ -1199,6 +1283,46 @@ export class MongoSyncBucketStorage }; } + private async getParameterBucketChangesV3( + options: GetCheckpointChangesOptions + ): Promise> { + const limit = 1000; + const parameterUpdates: { lookup: bson.Binary }[] = []; + + for (const collection of await this.db.listBucketParameterCollectionsV3(this.group_id)) { + if (parameterUpdates.length > limit) { + break; + } + + const remaining = limit + 1 - parameterUpdates.length; + const updates = await collection + .find( + { + _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint } + }, + { + projection: { + lookup: 1 + }, + limit: remaining, + batchSize: remaining + 1, + singleBatch: true + } + ) + .toArray(); + parameterUpdates.push(...updates); + } + + const invalidateParameterUpdates = parameterUpdates.length > limit; + + return { + invalidateParameterBuckets: invalidateParameterUpdates, + updatedParameterLookups: invalidateParameterUpdates + ? new Set() + : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup)))) + }; + } + // If we processed all connections together for each checkpoint, we could do a single lookup for all connections. // In practice, specific connections may fall behind. So instead, we just cache the results of each specific lookup. // TODO (later): diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index 426b2bc86..bfe8cec84 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -9,10 +9,10 @@ import { VersionedPowerSyncMongo } from './db.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { BucketStateDocument, - CommonBucketParameterDocument, CommonCurrentBucket, CommonCurrentLookup, SourceKey, + TaggedBucketParameterDocument, TaggedBucketDataDocument } from './models.js'; import { BucketDefinitionId } from './BucketDefinitionMapping.js'; @@ -75,7 +75,7 @@ export interface PersistedBatchOptions { export abstract class PersistedBatch { logger: Logger; bucketData: TaggedBucketDataDocument[] = []; - bucketParameters: mongo.AnyBulkWriteOperation[] = []; + bucketParameters: TaggedBucketParameterDocument[] = []; bucketStates: Map = new Map(); /** @@ -113,6 +113,8 @@ export abstract class PersistedBatch { protected abstract flushBucketData(session: mongo.ClientSession): Promise; + protected abstract flushBucketParameters(session: mongo.ClientSession): Promise; + protected abstract flushCurrentData(session: mongo.ClientSession): Promise; protected abstract resetCurrentData(): void; @@ -136,10 +138,6 @@ export abstract class PersistedBatch { } } - protected flushBucketParameters() { - return this.bucketParameters.length > 0; - } - protected addBucketDataPut(options: { op_id: InternalOpId; definitionId: BucketDefinitionId; @@ -210,12 +208,9 @@ export abstract class PersistedBatch { flushedSomething = true; await this.flushBucketData(session); } - if (this.flushBucketParameters()) { + if (this.bucketParameters.length > 0) { flushedSomething = true; - await db.bucket_parameters.bulkWrite(this.bucketParameters, { - session, - ordered: false - }); + await this.flushBucketParameters(session); } if (this.currentDataCount > 0) { flushedSomething = true; diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index d3cfdd1ad..f912f3c37 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -16,7 +16,9 @@ import { CurrentBucket, CurrentDataDocument, LEGACY_BUCKET_DATA_DEFINITION_ID, + LEGACY_BUCKET_PARAMETER_INDEX_ID, SourceKey, + taggedBucketParameterDocumentToV1, taggedBucketDataDocumentToV1, isCurrentBucketV3, isRecordedLookupV3 @@ -119,9 +121,8 @@ export class PersistedBatchV1 extends PersistedBatch { bucket_parameters: result.bucketParameters }; this.bucketParameters.push({ - insertOne: { - document: values - } + ...values, + index: LEGACY_BUCKET_PARAMETER_INDEX_ID }); this.currentSize += 200; @@ -141,9 +142,8 @@ export class PersistedBatchV1 extends PersistedBatch { bucket_parameters: [] }; this.bucketParameters.push({ - insertOne: { - document: values - } + ...values, + index: LEGACY_BUCKET_PARAMETER_INDEX_ID }); this.currentSize += 200; @@ -212,6 +212,20 @@ export class PersistedBatchV1 extends PersistedBatch { ); } + protected async flushBucketParameters(session: mongo.ClientSession) { + await this.db.v1_bucket_parameters.bulkWrite( + this.bucketParameters.map((document) => ({ + insertOne: { + document: taggedBucketParameterDocumentToV1(document) + } + })), + { + session, + ordered: false + } + ); + } + protected async flushCurrentData(session: mongo.ClientSession) { await this.db.v1_current_data.bulkWrite(this.currentData, { session, diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index ba9b2d4e1..120b75a4b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -19,6 +19,7 @@ import { isRecordedLookupV3, RecordedLookupV3, SourceKey, + taggedBucketParameterDocumentToV3, taggedBucketDataDocumentToV3 } from './models.js'; @@ -112,7 +113,6 @@ export class PersistedBatchV3 extends PersistedBatch { this.debugLastOpId = op_id; const values: BucketParameterDocumentV3 = { _id: op_id, - index: sourceDefinitionId, key: { g: this.group_id, t: mongoTableId(sourceTable.id), @@ -122,9 +122,8 @@ export class PersistedBatchV3 extends PersistedBatch { bucket_parameters: result.bucketParameters }; this.bucketParameters.push({ - insertOne: { - document: values - } + ...values, + index: sourceDefinitionId }); this.currentSize += 200; @@ -135,7 +134,6 @@ export class PersistedBatchV3 extends PersistedBatch { this.debugLastOpId = op_id; const values: BucketParameterDocumentV3 = { _id: op_id, - index: lookup.i, key: { g: this.group_id, t: mongoTableId(sourceTable.id), @@ -145,9 +143,8 @@ export class PersistedBatchV3 extends PersistedBatch { bucket_parameters: [] }; this.bucketParameters.push({ - insertOne: { - document: values - } + ...values, + index: lookup.i }); this.currentSize += 200; @@ -239,6 +236,30 @@ export class PersistedBatchV3 extends PersistedBatch { } } + protected async flushBucketParameters(session: mongo.ClientSession) { + const operationsByIndex = new Map(); + for (const document of this.bucketParameters) { + const existing = operationsByIndex.get(document.index) ?? []; + existing.push(document); + operationsByIndex.set(document.index, existing); + } + + for (const [indexId, documents] of operationsByIndex.entries()) { + await this.db.initializeBucketParameterCollectionV3(this.group_id, indexId); + await this.db.bucket_parameters_v3(this.group_id, indexId).bulkWrite( + documents.map((document) => ({ + insertOne: { + document: taggedBucketParameterDocumentToV3(document) + } + })), + { + session, + ordered: false + } + ); + } + } + protected async flushCurrentData(session: mongo.ClientSession) { await this.db.v3_current_data.bulkWrite(this.currentData, { session, diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index a3498c180..ea6a34137 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -11,7 +11,6 @@ import { BucketStateDocument, CheckpointEventDocument, ClientConnectionDocument, - CommonBucketParameterDocument, CommonCurrentDataDocument, CommonSourceTableDocument, CurrentDataDocument, @@ -26,7 +25,7 @@ import { WriteCheckpointDocument } from './models.js'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { BucketDefinitionId, BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { BucketDefinitionId, BucketDefinitionMapping, ParameterIndexId } from './BucketDefinitionMapping.js'; export interface PowerSyncMongoOptions { /** @@ -40,7 +39,6 @@ export class PowerSyncMongo { readonly v3_current_data: mongo.Collection; readonly bucket_data: mongo.Collection; readonly bucket_parameters: mongo.Collection; - readonly v3_bucket_parameters: mongo.Collection; readonly op_id_sequence: mongo.Collection; readonly sync_rules: mongo.Collection; readonly source_tables: mongo.Collection; @@ -68,7 +66,6 @@ export class PowerSyncMongo { this.v3_current_data = db.collection('v3_current_data'); this.bucket_data = db.collection('bucket_data'); this.bucket_parameters = db.collection('bucket_parameters'); - this.v3_bucket_parameters = db.collection('v3_bucket_parameters'); this.op_id_sequence = db.collection('op_id_sequence'); this.sync_rules = db.collection('sync_rules'); this.source_tables = db.collection('source_tables'); @@ -103,6 +100,35 @@ export class PowerSyncMongo { .map((collection) => this.db.collection(collection.name)); } + bucketParameterCollectionNameV3(groupId: number, indexId: ParameterIndexId) { + return `bucket_parameters_${groupId}_${indexId}`; + } + + bucketParametersV3(groupId: number, indexId: ParameterIndexId): mongo.Collection { + return this.db.collection(this.bucketParameterCollectionNameV3(groupId, indexId)); + } + + async listBucketParameterCollectionsV3(groupId?: number): Promise[]> { + const prefix = groupId == null ? 'bucket_parameters_' : `bucket_parameters_${groupId}_`; + const collections = await this.db.listCollections({}, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); + } + + async initializeBucketParameterCollectionV3(groupId: number, indexId: ParameterIndexId) { + await this.bucketParametersV3(groupId, indexId).createIndex( + { + lookup: 1, + _id: 1 + }, + { + name: 'lookup_op_id' + } + ); + } + /** * Clear all collections. */ @@ -114,7 +140,9 @@ export class PowerSyncMongo { await collection.drop(); } await this.bucket_parameters.deleteMany({}); - await this.v3_bucket_parameters.deleteMany({}); + for (const collection of await this.listBucketParameterCollectionsV3()) { + await collection.drop(); + } await this.op_id_sequence.deleteMany({}); await this.sync_rules.deleteMany({}); await this.source_tables.deleteMany({}); @@ -229,16 +257,6 @@ export class PowerSyncMongo { name: 'pending_delete' } ); - await this.v3_bucket_parameters.createIndex( - { - 'key.g': 1, - lookup: 1, - _id: 1 - }, - { - name: 'lookup_group_id' - } - ); await this.v3_source_tables.createIndex( { group_id: 1, @@ -334,12 +352,40 @@ export class VersionedPowerSyncMongo { return this.#upstream.listBucketDataCollectionsV3(groupId); } - get bucket_parameters() { + get v1_bucket_parameters() { if (this.storageConfig.incrementalReprocessing) { - return this.#upstream.v3_bucket_parameters as unknown as mongo.Collection; - } else { - return this.#upstream.bucket_parameters as unknown as mongo.Collection; + throw new ServiceAssertionError( + 'bucket_parameters collection should not be used when incrementalReprocessing is enabled' + ); + } + return this.#upstream.bucket_parameters; + } + + bucket_parameters_v3(groupId: number, indexId: ParameterIndexId) { + if (!this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' + ); + } + return this.#upstream.bucketParametersV3(groupId, indexId); + } + + listBucketParameterCollectionsV3(groupId?: number) { + if (!this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' + ); + } + return this.#upstream.listBucketParameterCollectionsV3(groupId); + } + + initializeBucketParameterCollectionV3(groupId: number, indexId: ParameterIndexId) { + if (!this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' + ); } + return this.#upstream.initializeBucketParameterCollectionV3(groupId, indexId); } get op_id_sequence() { diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 900c5b4e1..973f3c281 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -81,7 +81,9 @@ export interface BucketParameterDocument { bucket_parameters: Record[]; } -export interface BucketParameterDocumentV3 extends BucketParameterDocument { +export interface BucketParameterDocumentV3 extends BucketParameterDocument {} + +export interface TaggedBucketParameterDocument extends BucketParameterDocumentV3 { index: ParameterIndexId; } @@ -117,6 +119,11 @@ export interface TaggedBucketDataDocument extends BucketDataProperties { */ export const LEGACY_BUCKET_DATA_DEFINITION_ID = '0'; +/** + * Internal-only tag used for v1 bucket_parameters rows before they are converted to the v1 on-disk shape. + */ +export const LEGACY_BUCKET_PARAMETER_INDEX_ID = '0'; + export function bucketDataDocumentToTagged( document: BucketDataDocumentV1 | BucketDataDocumentV3, definitionId: BucketDefinitionId @@ -151,6 +158,26 @@ export function taggedBucketDataDocumentToV3(document: TaggedBucketDataDocument) return rest; } +export function bucketParameterDocumentToTagged( + document: BucketParameterDocument | BucketParameterDocumentV3, + index: ParameterIndexId +): TaggedBucketParameterDocument { + return { + ...document, + index + }; +} + +export function taggedBucketParameterDocumentToV1(document: TaggedBucketParameterDocument): BucketParameterDocument { + const { index: _index, ...rest } = document; + return rest; +} + +export function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3 { + const { index: _index, ...rest } = document; + return rest; +} + export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; export interface SourceTableDocument { @@ -383,7 +410,6 @@ export interface ClientConnectionDocument extends event_types.ClientConnection { export type CommonCurrentDataDocument = CurrentDataDocument | CurrentDataDocumentV3; export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3; export type CommonCurrentLookup = bson.Binary | RecordedLookupV3; -export type CommonBucketParameterDocument = BucketParameterDocument | BucketParameterDocumentV3; export type CommonSourceTableDocument = SourceTableDocument | SourceTableDocumentV3; export function isCurrentBucketV3(bucket: CommonCurrentBucket): bucket is CurrentBucketV3 { From dd2cc4ccd783b1b73cbbcd7ec3d1f4c8e62b8c0e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 16 Mar 2026 17:56:01 +0200 Subject: [PATCH 18/93] Initialize collections upfront. --- .../implementation/BucketDefinitionMapping.ts | 4 ++++ .../implementation/MongoSyncBucketStorage.ts | 13 ++++++++++++ .../implementation/PersistedBatchV3.ts | 1 - .../src/storage/implementation/db.ts | 21 ------------------- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts index 586a622e4..1aadd36d4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -46,6 +46,10 @@ export class BucketDefinitionMapping { return Object.values(this.definitions); } + allParameterIndexIds(): ParameterIndexId[] { + return Object.values(this.parameterLookupMapping); + } + parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId { const key = `${source.defaultLookupScope.lookupName}#${source.defaultLookupScope.queryId}`; const defId = this.parameterLookupMapping[key]; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index ed0f4ac0a..b85af86d0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -183,6 +183,18 @@ export class MongoSyncBucketStorage const collection = this.db.bucket_data_v3(this.group_id, source).collectionName; await this.db.db.createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }); } + for (let indexId of mapping.allParameterIndexIds()) { + await this.db.bucket_parameters_v3(this.group_id, indexId).createIndex( + { + lookup: 1, + key: 1, + _id: -1 + }, + { + name: 'lookup_op_id' + } + ); + } this.#storageInitialized = true; } @@ -448,6 +460,7 @@ export class MongoSyncBucketStorage } const groupedParameters: SqliteJsonRow[][] = []; + // FIXME: Optimize these lookups, properly utilizing the new index on {lookup: 1, key: 1, _id: -1}. for (const [indexId, lookupFilter] of lookupsByIndex.entries()) { const rows = await this.db .bucket_parameters_v3(this.group_id, indexId) diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 120b75a4b..29a8310b2 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -245,7 +245,6 @@ export class PersistedBatchV3 extends PersistedBatch { } for (const [indexId, documents] of operationsByIndex.entries()) { - await this.db.initializeBucketParameterCollectionV3(this.group_id, indexId); await this.db.bucket_parameters_v3(this.group_id, indexId).bulkWrite( documents.map((document) => ({ insertOne: { diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index ea6a34137..2847b0ee9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -117,18 +117,6 @@ export class PowerSyncMongo { .map((collection) => this.db.collection(collection.name)); } - async initializeBucketParameterCollectionV3(groupId: number, indexId: ParameterIndexId) { - await this.bucketParametersV3(groupId, indexId).createIndex( - { - lookup: 1, - _id: 1 - }, - { - name: 'lookup_op_id' - } - ); - } - /** * Clear all collections. */ @@ -379,15 +367,6 @@ export class VersionedPowerSyncMongo { return this.#upstream.listBucketParameterCollectionsV3(groupId); } - initializeBucketParameterCollectionV3(groupId: number, indexId: ParameterIndexId) { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' - ); - } - return this.#upstream.initializeBucketParameterCollectionV3(groupId, indexId); - } - get op_id_sequence() { return this.#upstream.op_id_sequence; } From d9634fcab2f8a7abbaee29b48f33a774a011209c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Mar 2026 10:49:24 +0200 Subject: [PATCH 19/93] Workaround for MongoDB SERVER-121822. --- .../storage/implementation/MongoCompactor.ts | 105 ++++++++---------- 1 file changed, 45 insertions(+), 60 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 78553db1f..358f1cc14 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -186,45 +186,44 @@ export class MongoCompactor { }; // Constant lower bound - const lowerBound = this.bucketDataKey({ - _id: { b: bucket, o: new mongo.MinKey() as any }, - def: bucketCollection.definitionId - }); + const lowerBound = this.bucketDataKey(bucket, new mongo.MinKey() as any); // Upper bound is adjusted for each batch - let upperBound = this.bucketDataKey({ - _id: { b: bucket, o: new mongo.MaxKey() as any }, - def: bucketCollection.definitionId - }); + let upperBound = this.bucketDataKey(bucket, new mongo.MaxKey() as any); try { while (!this.signal?.aborted) { // Query one batch at a time, to avoid cursor timeouts - const cursor = bucketCollection.collection.aggregate( - [ - { - $match: { - _id: { - $gte: lowerBound, - $lt: upperBound - } - } - }, - { $sort: { _id: -1 } }, - { $limit: this.moveBatchQueryLimit }, - { - $project: { - _id: 1, - op: 1, - table: 1, - row_id: 1, - source_table: 1, - source_key: 1, - checksum: 1, - size: { $bsonSize: '$$ROOT' } - } + const pipeline = [ + { + $match: { + _id: { + $gte: lowerBound, + $lt: upperBound + }, + // Workaround for bug with clustered collections (storage v3), where the $lt operator + // may include the upperBound. + // https://jira.mongodb.org/browse/SERVER-121822 + '_id.o': { $lt: upperBound.o } } - ], + }, + { $sort: { _id: -1 } }, + { $limit: this.moveBatchQueryLimit }, + { + $project: { + _id: 1, + op: 1, + table: 1, + row_id: 1, + source_table: 1, + source_key: 1, + checksum: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ]; + const cursor = bucketCollection.collection.aggregate( + pipeline, { // batchSize is 1 more than limit to auto-close the cursor. // See https://github.com/mongodb/node-mongodb-native/pull/4580 @@ -241,8 +240,8 @@ export class MongoCompactor { break; } - // Set upperBound for the next batch - upperBound = this.bucketDataKey(batch[batch.length - 1]); + // Reuse the exact collection _id value from Mongo for the next bound + upperBound = rawBatch[rawBatch.length - 1]._id; for (let doc of batch) { if (doc._id.o > this.maxOpId) { @@ -264,7 +263,7 @@ export class MongoCompactor { this.updates.push({ updateOne: { - filter: { _id: this.bucketDataKey(doc) }, + filter: { _id: this.bucketDataKey(doc._id.b, doc._id.o) }, update: { $set: { op: 'MOVE', @@ -420,11 +419,8 @@ export class MongoCompactor { const opFilter = { _id: { - $gte: this.bucketDataKey({ - _id: { b: bucket, o: new mongo.MinKey() as any }, - def: this.activeBucketDefinitionId - }), - $lte: this.bucketDataKey({ _id: { b: bucket, o: clearOp }, def: this.activeBucketDefinitionId }) + $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), + $lte: this.bucketDataKey(bucket, clearOp) } }; @@ -487,11 +483,8 @@ export class MongoCompactor { await bucketCollection.deleteMany( { _id: { - $gte: this.bucketDataKey({ - _id: { b: bucket, o: new mongo.MinKey() as any }, - def: this.activeBucketDefinitionId - }), - $lte: this.bucketDataKey(lastOp!) + $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), + $lte: this.bucketDataKey(lastOp!._id.b, lastOp!._id.o) } } as any, { session } as any @@ -749,24 +742,16 @@ export class MongoCompactor { await this.flush(); } - private bucketDataKey(document: Pick) { + private bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey) { if (this.db.storageConfig.incrementalReprocessing) { - return taggedBucketDataDocumentToV3({ - def: document.def, - _id: document._id, - op: 'CLEAR', - checksum: 0n, - data: null - })._id; + return { b: bucket, o: opId as any }; } - return taggedBucketDataDocumentToV1(this.group_id, { - def: document.def, - _id: document._id, - op: 'CLEAR', - checksum: 0n, - data: null - })._id; + return { + g: this.group_id, + b: bucket, + o: opId as any + }; } private async getBucketDataCollection( From 82089ce51ee1e99413b119f8a35323c27d32bf4a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Mar 2026 10:53:44 +0200 Subject: [PATCH 20/93] Update snapshots. --- .../test/src/__snapshots__/storage_sync.test.ts.snap | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap index 70387ff0e..d59bfb500 100644 --- a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -3284,5 +3284,3 @@ exports[`sync - mongodb > storage v3 > sync updates to parameter query only 2`] }, ] `; - -exports[`sync - mongodb > storage v3 > write checkpoint 1`] = `[]`; From df12101367835173630cf8b8b2a8291f498a0a19 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Mar 2026 11:31:40 +0200 Subject: [PATCH 21/93] Fix for parameter lookups. --- .../implementation/BucketDefinitionMapping.ts | 15 ++++++++++++++- .../implementation/MongoSyncBucketStorage.ts | 6 +++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts index 1aadd36d4..9740eaaac 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -51,7 +51,7 @@ export class BucketDefinitionMapping { } parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId { - const key = `${source.defaultLookupScope.lookupName}#${source.defaultLookupScope.queryId}`; + const key = this.parameterLookupKey(source.defaultLookupScope.lookupName, source.defaultLookupScope.queryId); const defId = this.parameterLookupMapping[key]; if (defId == null) { throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`); @@ -59,6 +59,19 @@ export class BucketDefinitionMapping { return defId; } + parameterLookupScopeId(scope: Pick) { + const key = this.parameterLookupKey(scope.lookupName, scope.queryId); + const defId = this.parameterLookupMapping[key]; + if (defId == null) { + throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`); + } + return defId; + } + + private parameterLookupKey(lookupName: string, queryId: string) { + return `${lookupName}#${queryId}`; + } + serialize(): NonNullable { return { definitions: { ...this.definitions }, diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index b85af86d0..9c11ef2de 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -453,7 +453,11 @@ export class MongoSyncBucketStorage const lookupsByIndex = new Map(); for (const lookup of lookups) { - const indexId = this.sync_rules.mapping.parameterLookupId(lookup.source); + const [lookupName, queryId] = lookup.values; + if (typeof lookupName != 'string' || typeof queryId != 'string') { + throw new ServiceAssertionError('Invalid scoped parameter lookup identifier'); + } + const indexId = this.sync_rules.mapping.parameterLookupScopeId({ lookupName, queryId }); const existing = lookupsByIndex.get(indexId) ?? []; existing.push(storage.serializeLookup(lookup)); lookupsByIndex.set(indexId, existing); From c9f4780b30c7b93afe88fc305f40b14e220b8d32 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 17 Mar 2026 11:46:40 +0200 Subject: [PATCH 22/93] Optimize parameter query lookups. --- .../implementation/MongoSyncBucketStorage.ts | 117 +++++++++++------- 1 file changed, 75 insertions(+), 42 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 9c11ef2de..bf8eab580 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -36,6 +36,7 @@ import { BucketDataDocumentV1, BucketDataKeyV1, BucketDataDocumentV3, + BucketParameterDocumentV3, BucketStateDocument, CommonSourceTableDocument, LEGACY_BUCKET_DATA_DEFINITION_ID, @@ -451,60 +452,92 @@ export class MongoSyncBucketStorage return this.db.client.withSession({ snapshot: true }, async (session) => { setSessionSnapshotTime(session, checkpoint.snapshotTime); - const lookupsByIndex = new Map(); - for (const lookup of lookups) { + // Conceptually we do each lookup separately as an aggregation pipeline. We then + // use $unionWith to combine it all into a single operation. + // This helps to: + // 1. Handle different collections in the same query (although this may not be common in practice). + // 2. Efficiently use the index to get the first item grouped by {lookup, key}. + // The index is on { lookup: 1, key: 1, _id: -1 }. + + const buildLookupPipeline = ( + lookup: ScopedParameterLookup + ): { + collection: mongo.Collection; + pipeline: mongo.Document[]; + } => { const [lookupName, queryId] = lookup.values; if (typeof lookupName != 'string' || typeof queryId != 'string') { throw new ServiceAssertionError('Invalid scoped parameter lookup identifier'); } const indexId = this.sync_rules.mapping.parameterLookupScopeId({ lookupName, queryId }); - const existing = lookupsByIndex.get(indexId) ?? []; - existing.push(storage.serializeLookup(lookup)); - lookupsByIndex.set(indexId, existing); - } - - const groupedParameters: SqliteJsonRow[][] = []; - // FIXME: Optimize these lookups, properly utilizing the new index on {lookup: 1, key: 1, _id: -1}. - for (const [indexId, lookupFilter] of lookupsByIndex.entries()) { - const rows = await this.db - .bucket_parameters_v3(this.group_id, indexId) - .aggregate( - [ - { - $match: { - lookup: { $in: lookupFilter }, - _id: { $lte: checkpoint.checkpoint } - } - }, - { - $sort: { - _id: -1 - } - }, - { - $group: { - _id: { key: '$key', lookup: '$lookup' }, - bucket_parameters: { - $first: '$bucket_parameters' - } + const collection = this.db.bucket_parameters_v3(this.group_id, indexId); + const lookupFilter = storage.serializeLookup(lookup); + return { + collection, + pipeline: [ + { + $match: { + lookup: lookupFilter, + _id: { $lte: checkpoint.checkpoint } + } + }, + { + $sort: { + key: 1, + _id: -1 + } + }, + { + $group: { + _id: { + key: '$key' + }, + bucket_parameters: { + $first: '$bucket_parameters' } } - ], + }, { - session, - readConcern: 'snapshot', - maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + $project: { + _id: 0, + bucket_parameters: 1 + } } - ) - .toArray() - .catch((e) => { - throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); - }); + ] + }; + }; - groupedParameters.push(...rows.map((row) => row.bucket_parameters)); + const [firstLookup, ...remainingLookups] = lookups; + const firstQuery = firstLookup == null ? null : buildLookupPipeline(firstLookup); + if (firstQuery == null) { + return []; } - return groupedParameters.flat(); + const pipeline: mongo.Document[] = [ + ...firstQuery.pipeline, + ...remainingLookups.map((lookup) => { + const query = buildLookupPipeline(lookup); + return { + $unionWith: { + coll: query.collection.collectionName, + pipeline: query.pipeline + } + }; + }) + ]; + + const rows = await firstQuery.collection + .aggregate<{ bucket_parameters: SqliteJsonRow[] }>(pipeline, { + session, + readConcern: 'snapshot', + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + }) + .toArray() + .catch((e) => { + throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); + }); + + return rows.flatMap((row) => row.bucket_parameters); }); } From bc8968f827db892627168cceb0946398949ad810 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 11:01:13 +0200 Subject: [PATCH 23/93] Minor restructuring. --- .../src/storage/MongoBucketStorage.ts | 2 +- .../src/storage/implementation/db.ts | 31 +++++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 413e616e9..53844f61a 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -347,7 +347,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { .toArray() .catch(ignoreNotExisting); const v3_parameter_aggregates = await Promise.all( - (await this.db.listBucketParameterCollectionsV3()).map((collection) => + (await this.db.listParameterIndexCollectionsV3()).map((collection) => collection .aggregate([ { diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 2847b0ee9..b41433e5a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -100,17 +100,28 @@ export class PowerSyncMongo { .map((collection) => this.db.collection(collection.name)); } - bucketParameterCollectionNameV3(groupId: number, indexId: ParameterIndexId) { - return `bucket_parameters_${groupId}_${indexId}`; + bucketParameterCollectionNameV3(replicationStreamId: number, indexId: ParameterIndexId) { + return `parameter_index_${replicationStreamId}_${indexId}`; } - bucketParametersV3(groupId: number, indexId: ParameterIndexId): mongo.Collection { - return this.db.collection(this.bucketParameterCollectionNameV3(groupId, indexId)); + parameterIndexV3( + replicationStreamId: number, + indexId: ParameterIndexId + ): mongo.Collection { + return this.db.collection(this.bucketParameterCollectionNameV3(replicationStreamId, indexId)); } - async listBucketParameterCollectionsV3(groupId?: number): Promise[]> { - const prefix = groupId == null ? 'bucket_parameters_' : `bucket_parameters_${groupId}_`; - const collections = await this.db.listCollections({}, { nameOnly: true }).toArray(); + /** + * List parameter index collections. + * + * @param replicationStreamId null only to list all collections in the db for clearing + * @returns + */ + async listParameterIndexCollectionsV3( + replicationStreamId?: number + ): Promise[]> { + const prefix = replicationStreamId == null ? `parameter_index_` : `parameter_index_${replicationStreamId}_`; + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); return collections .filter((collection) => collection.name.startsWith(prefix)) @@ -128,7 +139,7 @@ export class PowerSyncMongo { await collection.drop(); } await this.bucket_parameters.deleteMany({}); - for (const collection of await this.listBucketParameterCollectionsV3()) { + for (const collection of await this.listParameterIndexCollectionsV3()) { await collection.drop(); } await this.op_id_sequence.deleteMany({}); @@ -355,7 +366,7 @@ export class VersionedPowerSyncMongo { 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' ); } - return this.#upstream.bucketParametersV3(groupId, indexId); + return this.#upstream.parameterIndexV3(groupId, indexId); } listBucketParameterCollectionsV3(groupId?: number) { @@ -364,7 +375,7 @@ export class VersionedPowerSyncMongo { 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' ); } - return this.#upstream.listBucketParameterCollectionsV3(groupId); + return this.#upstream.listParameterIndexCollectionsV3(groupId); } get op_id_sequence() { From 421a8ea63bc917fd934a08ba6b52512ff9ed84c8 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 11:13:39 +0200 Subject: [PATCH 24/93] Split current_data into separate source_record_ collections. --- .../src/storage/MongoBucketStorage.ts | 41 ++++----- .../implementation/MongoBucketBatch.ts | 89 +++++++++++-------- .../implementation/MongoBucketBatchV3.ts | 16 ++-- .../implementation/MongoSyncBucketStorage.ts | 19 ++-- .../implementation/PersistedBatchV1.ts | 41 ++++++++- .../implementation/PersistedBatchV3.ts | 42 ++++++++- .../src/storage/implementation/db.ts | 85 +++++++++++++----- .../test/src/storage_sync.test.ts | 7 +- 8 files changed, 235 insertions(+), 105 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 53844f61a..9c50e8110 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -361,27 +361,20 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ) ); - const v1_replication_aggregate = await this.db.current_data - .aggregate([ - { - $collStats: { - storageStats: {} - } - } - ]) - .toArray() - .catch(ignoreNotExisting); - - const v3_replication_aggregate = await this.db.v3_current_data - .aggregate([ - { - $collStats: { - storageStats: {} - } - } - ]) - .toArray() - .catch(ignoreNotExisting); + const source_record_aggregates = await Promise.all( + (await this.db.listSourceRecordCollections()).map((collection) => + collection + .aggregate([ + { + $collStats: { + storageStats: {} + } + } + ]) + .toArray() + .catch(ignoreNotExisting) + ) + ); return { operations_size_bytes: Number(operations_aggregate[0].storageStats.size) + @@ -389,8 +382,10 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size) + v3_parameter_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0), - replication_size_bytes: - Number(v1_replication_aggregate[0].storageStats.size) + Number(v3_replication_aggregate[0].storageStats.size) + replication_size_bytes: source_record_aggregates.reduce( + (total, aggregate) => total + Number(aggregate[0]?.storageStats?.size ?? 0), + 0 + ) }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 69c2ed13e..f4608ebfb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -244,26 +244,29 @@ export abstract class MongoBucketBatch sizes = new Map(); - const sizeCursor: mongo.AggregationCursor<{ _id: SourceKey; size: number }> = - this.db.common_current_data.aggregate( - [ - { - $match: { - _id: { $in: sizeLookups } - } - }, - { - $project: { - _id: 1, - size: { $bsonSize: '$$ROOT' } + for (const [sourceTableId, sourceKeys] of this.groupSourceKeysByTable(sizeLookups)) { + const sizeCursor: mongo.AggregationCursor<{ _id: SourceKey; size: number }> = this.db + .common_current_data(this.group_id, sourceTableId) + .aggregate( + [ + { + $match: { + _id: { $in: sourceKeys } + } + }, + { + $project: { + _id: 1, + size: { $bsonSize: '$$ROOT' } + } } - } - ], - { session } - ); - for await (let doc of sizeCursor.stream()) { - const key = cacheKey(doc._id.t, doc._id.k); - sizes.set(key, doc.size); + ], + { session } + ); + for await (let doc of sizeCursor.stream()) { + const key = cacheKey(doc._id.t, doc._id.k); + sizes.set(key, doc.size); + } } } @@ -288,14 +291,16 @@ export abstract class MongoBucketBatch let current_data_lookup = new Map(); // With skipExistingRows, we only need to know whether or not the row exists. const projection = this.skipExistingRows ? { _id: 1 } : undefined; - const cursor = this.db.common_current_data.find( - { - _id: { $in: lookups } - }, - { session, projection } - ); - for await (let doc of cursor.stream()) { - current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc); + for (const [sourceTableId, sourceKeys] of this.groupSourceKeysByTable(lookups)) { + const cursor = this.db.common_current_data(this.group_id, sourceTableId).find( + { + _id: { $in: sourceKeys } + }, + { session, projection } + ); + for await (let doc of cursor.stream()) { + current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc); + } } let persistedBatch: PersistedBatch | null = this.createPersistedBatch(transactionSize); @@ -1029,15 +1034,17 @@ export abstract class MongoBucketBatch pending_delete: { $exists: false } }; - const cursor = this.db.common_current_data.find(current_data_filter, { - projection: { - _id: 1, - buckets: 1, - lookups: 1 - }, - limit: BATCH_LIMIT, - session: session - }); + const cursor = this.db + .common_current_data(this.group_id, mongoTableId(sourceTable.id)) + .find(current_data_filter, { + projection: { + _id: 1, + buckets: 1, + lookups: 1 + }, + limit: BATCH_LIMIT, + session: session + }); const batch = await cursor.toArray(); const persistedBatch = this.createPersistedBatch(0); @@ -1203,4 +1210,14 @@ export abstract class MongoBucketBatch [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table)) ); } + + private groupSourceKeysByTable(sourceKeys: SourceKey[]): Map { + const grouped = new Map(); + for (const sourceKey of sourceKeys) { + const existing = grouped.get(sourceKey.t) ?? []; + existing.push(sourceKey); + grouped.set(sourceKey.t, existing); + } + return grouped; + } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts index 5b593dae6..4bfd22f3f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts @@ -75,13 +75,17 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { } protected async cleanupCurrentData(lastCheckpoint: bigint): Promise { - const result = await this.db.v3_current_data.deleteMany({ - '_id.g': this.group_id, - pending_delete: { $exists: true, $lte: lastCheckpoint } - }); - if (result.deletedCount > 0) { + let deletedCount = 0; + for (const collection of await this.db.listCommonCurrentDataCollections(this.group_id)) { + const result = await collection.deleteMany({ + '_id.g': this.group_id, + pending_delete: { $exists: true, $lte: lastCheckpoint } + }); + deletedCount += result.deletedCount; + } + if (deletedCount > 0) { this.logger.info( - `Cleaned up ${result.deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}` + `Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}` ); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index bf8eab580..a995eee07 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -182,7 +182,14 @@ export class MongoSyncBucketStorage const mapping = this.sync_rules.mapping; for (let source of mapping.allBucketDefinitionIds()) { const collection = this.db.bucket_data_v3(this.group_id, source).collectionName; - await this.db.db.createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }); + await this.db.db + .createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') { + return; + } + throw error; + }); } for (let indexId of mapping.allParameterIndexIds()) { await this.db.bucket_parameters_v3(this.group_id, indexId).createIndex( @@ -974,12 +981,9 @@ export class MongoSyncBucketStorage ); } - await this.db.common_current_data.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['t', 'k']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); + for (const collection of await this.db.listCommonCurrentDataCollections(this.group_id)) { + await collection.drop(); + } await this.db.bucket_state.deleteMany( { @@ -994,6 +998,7 @@ export class MongoSyncBucketStorage }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); + this.#storageInitialized = false; } async reportError(e: any): Promise { diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index f912f3c37..754633843 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -227,13 +227,46 @@ export class PersistedBatchV1 extends PersistedBatch { } protected async flushCurrentData(session: mongo.ClientSession) { - await this.db.v1_current_data.bulkWrite(this.currentData, { - session, - ordered: true - }); + const operationsBySourceTable = new Map(); + for (const operation of this.currentData) { + const sourceTableId = this.getSourceTableIdHex(operation); + if (sourceTableId == null) { + throw new ReplicationAssertionError('Missing source table id for current_data operation'); + } + const existing = operationsBySourceTable.get(sourceTableId) ?? []; + existing.push(operation); + operationsBySourceTable.set(sourceTableId, existing); + } + + for (const operations of operationsBySourceTable.values()) { + const firstOperation = operations[0]!; + const sourceTableId = this.getSourceTableId(firstOperation); + if (sourceTableId == null) { + throw new ReplicationAssertionError('Missing source table id for current_data bulkWrite'); + } + await this.db.initializeCurrentDataCollection(this.group_id, sourceTableId); + await this.db.v1_current_data(this.group_id, sourceTableId).bulkWrite(operations, { + session, + ordered: true + }); + } } protected resetCurrentData() { this.currentData = []; } + + private getSourceTableIdHex(operation: mongo.AnyBulkWriteOperation): string | undefined { + return this.getSourceTableId(operation)?.toHexString(); + } + + private getSourceTableId(operation: mongo.AnyBulkWriteOperation): bson.ObjectId | undefined { + if ('updateOne' in operation) { + return operation.updateOne.filter._id?.t; + } + if ('deleteOne' in operation) { + return operation.deleteOne.filter._id?.t; + } + return undefined; + } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 29a8310b2..dac4416ec 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -2,6 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; +import * as bson from 'bson'; import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; import { BucketDefinitionId } from './BucketDefinitionMapping.js'; @@ -260,13 +261,46 @@ export class PersistedBatchV3 extends PersistedBatch { } protected async flushCurrentData(session: mongo.ClientSession) { - await this.db.v3_current_data.bulkWrite(this.currentData, { - session, - ordered: true - }); + const operationsBySourceTable = new Map(); + for (const operation of this.currentData) { + const sourceTableId = this.getSourceTableIdHex(operation); + if (sourceTableId == null) { + throw new ReplicationAssertionError('Missing source table id for current_data operation'); + } + const existing = operationsBySourceTable.get(sourceTableId) ?? []; + existing.push(operation); + operationsBySourceTable.set(sourceTableId, existing); + } + + for (const operations of operationsBySourceTable.values()) { + const firstOperation = operations[0]!; + const sourceTableId = this.getSourceTableId(firstOperation); + if (sourceTableId == null) { + throw new ReplicationAssertionError('Missing source table id for current_data bulkWrite'); + } + await this.db.initializeCurrentDataCollection(this.group_id, sourceTableId); + await this.db.v3_current_data(this.group_id, sourceTableId).bulkWrite(operations, { + session, + ordered: true + }); + } } protected resetCurrentData() { this.currentData = []; } + + private getSourceTableIdHex(operation: mongo.AnyBulkWriteOperation): string | undefined { + return this.getSourceTableId(operation)?.toHexString(); + } + + private getSourceTableId(operation: mongo.AnyBulkWriteOperation): bson.ObjectId | undefined { + if ('updateOne' in operation) { + return operation.updateOne.filter._id?.t; + } + if ('deleteOne' in operation) { + return operation.deleteOne.filter._id?.t; + } + return undefined; + } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index b41433e5a..d881670b5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -128,12 +128,37 @@ export class PowerSyncMongo { .map((collection) => this.db.collection(collection.name)); } + sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + return `source_records_${replicationStreamId}_${sourceTableId.toHexString()}`; + } + + sourceRecords( + replicationStreamId: number, + sourceTableId: mongo.ObjectId + ): mongo.Collection { + return this.db.collection(this.sourceRecordsCollectionName(replicationStreamId, sourceTableId)); + } + + async listSourceRecordCollections( + replicationStreamId?: number + ): Promise[]> { + const prefix = replicationStreamId == null ? 'source_records_' : `source_records_${replicationStreamId}_`; + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); + } + /** * Clear all collections. */ async clear() { await this.current_data.deleteMany({}); await this.v3_current_data.deleteMany({}); + for (const collection of await this.listSourceRecordCollections()) { + await collection.drop(); + } await this.bucket_data.deleteMany({}); for (const collection of await this.listBucketDataCollectionsV3()) { await collection.drop(); @@ -244,18 +269,6 @@ export class PowerSyncMongo { async initializeStorageVersion(storageConfig: StorageConfig) { if (storageConfig.incrementalReprocessing) { - // Initialize the v3_current_data collection, which is used for the new storage version. - // No-op if this already exists - await this.v3_current_data.createIndex( - { - '_id.g': 1, - pending_delete: 1 - }, - { - partialFilterExpression: { pending_delete: { $exists: true } }, - name: 'pending_delete' - } - ); await this.v3_source_tables.createIndex( { group_id: 1, @@ -270,6 +283,27 @@ export class PowerSyncMongo { ); } } + + async initializeSourceRecordsCollection( + storageConfig: StorageConfig, + replicationStreamId: number, + sourceTableId: mongo.ObjectId + ) { + if (!storageConfig.incrementalReprocessing) { + return; + } + + await this.sourceRecords(replicationStreamId, sourceTableId).createIndex( + { + '_id.g': 1, + pending_delete: 1 + }, + { + partialFilterExpression: { pending_delete: { $exists: true } }, + name: 'pending_delete' + } + ); + } } /** @@ -294,30 +328,37 @@ export class VersionedPowerSyncMongo { * * Use in places where it does not matter which version is used. */ - get common_current_data(): mongo.Collection { - if (this.storageConfig.incrementalReprocessing) { - return this.#upstream.v3_current_data as unknown as mongo.Collection; - } else { - return this.#upstream.current_data as unknown as mongo.Collection; - } + common_current_data( + replicationStreamId: number, + sourceTableId: mongo.ObjectId + ): mongo.Collection { + return this.#upstream.sourceRecords(replicationStreamId, sourceTableId); } - get v1_current_data() { + v1_current_data(replicationStreamId: number, sourceTableId: mongo.ObjectId) { if (this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'current_data collection should not be used when incrementalReprocessing is enabled' ); } - return this.#upstream.current_data; + return this.#upstream.sourceRecords(replicationStreamId, sourceTableId); } - get v3_current_data() { + v3_current_data(replicationStreamId: number, sourceTableId: mongo.ObjectId) { if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'v3_current_data collection should not be used when incrementalReprocessing is disabled' ); } - return this.#upstream.v3_current_data; + return this.#upstream.sourceRecords(replicationStreamId, sourceTableId); + } + + listCommonCurrentDataCollections(replicationStreamId?: number) { + return this.#upstream.listSourceRecordCollections(replicationStreamId); + } + + initializeCurrentDataCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + return this.#upstream.initializeSourceRecordsCollection(this.storageConfig, replicationStreamId, sourceTableId); } get bucket_data() { diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 558a1f0c0..cdbced7d3 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -3,7 +3,7 @@ import { bucketRequest, register, test_utils } from '@powersync/service-core-tes import { describe, expect, test } from 'vitest'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; -import { CurrentDataDocumentV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; +import { CurrentBucketV3, CurrentDataDocumentV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) { register.registerSyncTests(storageConfig.factory, { @@ -180,8 +180,9 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor await writer.flush(); const mongoFactory = factory as MongoBucketStorage; - const currentData = await mongoFactory.db.v3_current_data.findOne({}); - const firstBucket: CurrentDataDocumentV3['buckets'][number] | undefined = currentData?.buckets[0]; + const currentDataCollections = await mongoFactory.db.listSourceRecordCollections(syncRules.id); + const currentData = await currentDataCollections[0]?.findOne({}); + const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined; expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/); const bucketCollections = await mongoFactory.db.db From 8c65154a23be2ec20adfa18c54cb0b4ffe7c001e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 11:45:24 +0200 Subject: [PATCH 25/93] Refactor _id for source_records. --- .../implementation/MongoBucketBatch.ts | 87 ++++++++------ .../implementation/MongoBucketBatchV1.ts | 37 +++++- .../implementation/MongoBucketBatchV3.ts | 30 ++++- .../implementation/MongoParameterCompactor.ts | 21 +++- .../storage/implementation/PersistedBatch.ts | 13 ++- .../implementation/PersistedBatchV1.ts | 6 +- .../implementation/PersistedBatchV3.ts | 106 ++++++++---------- .../src/storage/implementation/db.ts | 1 - .../src/storage/implementation/models.ts | 24 +++- 9 files changed, 207 insertions(+), 118 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index f4608ebfb..04dfb5011 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -37,6 +37,7 @@ import { CommonCurrentBucket, CommonCurrentLookup, CommonCurrentDataDocument, + CurrentDataDocumentId, SourceKey, SyncRuleDocument } from './models.js'; @@ -164,13 +165,29 @@ export abstract class MongoBucketBatch protected abstract mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CommonCurrentLookup[]; + protected abstract createCurrentDataId( + sourceTableId: bson.ObjectId, + replicaId: storage.ReplicaId + ): CurrentDataDocumentId; + protected abstract createCurrentDataDocument( - id: SourceKey, + id: CurrentDataDocumentId, data: bson.Binary, buckets: CommonCurrentBucket[], lookups: CommonCurrentLookup[] ): CommonCurrentDataDocument; + protected abstract createCurrentDataLookupFilter( + sourceTableId: bson.ObjectId, + replicaIds: storage.ReplicaId[] + ): mongo.Filter; + + protected abstract currentDataCacheKey(sourceTableId: bson.ObjectId, document: CommonCurrentDataDocument): string; + + protected abstract currentDataReplicaId(document: CommonCurrentDataDocument): storage.ReplicaId; + + protected abstract activeCurrentDataFilter(sourceTableId: bson.ObjectId): mongo.Filter; + protected abstract cleanupCurrentData(lastCheckpoint: bigint): Promise; async flush(options?: storage.BatchBucketFlushOptions): Promise { @@ -238,20 +255,21 @@ export abstract class MongoBucketBatch // (automatically limited to 48MB(?) per batch by MongoDB). The issue is that it changes // the order of processing, which then becomes really tricky to manage. // This now takes 2+ queries, but doesn't have any issues with order of operations. - const sizeLookups: SourceKey[] = batch.batch.map((r) => { - return { g: this.group_id, t: mongoTableId(r.record.sourceTable.id), k: r.beforeId }; - }); + const sizeLookups = batch.batch.map((r) => ({ + sourceTableId: mongoTableId(r.record.sourceTable.id), + replicaId: r.beforeId + })); sizes = new Map(); - for (const [sourceTableId, sourceKeys] of this.groupSourceKeysByTable(sizeLookups)) { - const sizeCursor: mongo.AggregationCursor<{ _id: SourceKey; size: number }> = this.db + for (const [sourceTableId, replicaIds] of this.groupReplicaIdsByTable(sizeLookups)) { + const sizeCursor: mongo.AggregationCursor = this.db .common_current_data(this.group_id, sourceTableId) .aggregate( [ { $match: { - _id: { $in: sourceKeys } + ...this.createCurrentDataLookupFilter(sourceTableId, replicaIds) } }, { @@ -264,7 +282,7 @@ export abstract class MongoBucketBatch { session } ); for await (let doc of sizeCursor.stream()) { - const key = cacheKey(doc._id.t, doc._id.k); + const key = this.currentDataCacheKey(sourceTableId, doc); sizes.set(key, doc.size); } } @@ -285,21 +303,19 @@ export abstract class MongoBucketBatch } continue; } - const lookups: SourceKey[] = b.map((r) => { - return { g: this.group_id, t: mongoTableId(r.record.sourceTable.id), k: r.beforeId }; - }); + const lookups = b.map((r) => ({ + sourceTableId: mongoTableId(r.record.sourceTable.id), + replicaId: r.beforeId + })); let current_data_lookup = new Map(); // With skipExistingRows, we only need to know whether or not the row exists. const projection = this.skipExistingRows ? { _id: 1 } : undefined; - for (const [sourceTableId, sourceKeys] of this.groupSourceKeysByTable(lookups)) { - const cursor = this.db.common_current_data(this.group_id, sourceTableId).find( - { - _id: { $in: sourceKeys } - }, - { session, projection } - ); + for (const [sourceTableId, replicaIds] of this.groupReplicaIdsByTable(lookups)) { + const cursor = this.db + .common_current_data(this.group_id, sourceTableId) + .find(this.createCurrentDataLookupFilter(sourceTableId, replicaIds), { session, projection }); for await (let doc of cursor.stream()) { - current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc); + current_data_lookup.set(this.currentDataCacheKey(sourceTableId, doc), doc); } } @@ -366,7 +382,8 @@ export abstract class MongoBucketBatch let existing_lookups: CommonCurrentLookup[] = []; let new_lookups: CommonCurrentLookup[] = []; - const before_key: SourceKey = { g: this.group_id, t: mongoTableId(record.sourceTable.id), k: beforeId }; + const sourceTableId = mongoTableId(record.sourceTable.id); + const before_key = this.createCurrentDataId(sourceTableId, beforeId); if (this.skipExistingRows) { if (record.tag == SaveOperationTag.INSERT) { @@ -569,8 +586,9 @@ export abstract class MongoBucketBatch // 5. TOAST: Update current data and bucket list. if (afterId) { // Insert or update - const after_key: SourceKey = { g: this.group_id, t: mongoTableId(sourceTable.id), k: afterId }; + const after_key = this.createCurrentDataId(sourceTableId, afterId); batch.upsertCurrentData({ + sourceTableId, id: after_key, data: afterData, buckets: new_buckets, @@ -584,7 +602,7 @@ export abstract class MongoBucketBatch // Note that this is a soft delete. // We don't specifically need a new or unique op_id here, but it must be greater than the // last checkpoint, so we use next(). - batch.softDeleteCurrentData(before_key, opSeq.next()); + batch.softDeleteCurrentData(sourceTableId, before_key, opSeq.next()); } return result; } @@ -1027,12 +1045,7 @@ export abstract class MongoBucketBatch let lastBatchCount = BATCH_LIMIT; while (lastBatchCount == BATCH_LIMIT) { await this.withReplicationTransaction(`Truncate ${sourceTable.qualifiedName}`, async (session, opSeq) => { - const current_data_filter: mongo.Filter = { - _id: idPrefixFilter({ g: this.group_id, t: mongoTableId(sourceTable.id) }, ['k']), - // Skip soft-deleted data - // Works for both v1 and v3 current_data schemas - pending_delete: { $exists: false } - }; + const current_data_filter = this.activeCurrentDataFilter(mongoTableId(sourceTable.id)); const cursor = this.db .common_current_data(this.group_id, mongoTableId(sourceTable.id)) @@ -1054,18 +1067,18 @@ export abstract class MongoBucketBatch before_buckets: value.buckets, evaluated: [], table: sourceTable, - sourceKey: value._id.k + sourceKey: this.currentDataReplicaId(value) }); persistedBatch.saveParameterData({ op_seq: opSeq, existing_lookups: value.lookups, evaluated: [], sourceTable: sourceTable, - sourceKey: value._id.k + sourceKey: this.currentDataReplicaId(value) }); // Since this is not from streaming replication, we can do a hard delete - persistedBatch.hardDeleteCurrentData(value._id); + persistedBatch.hardDeleteCurrentData(mongoTableId(sourceTable.id), value._id); } await persistedBatch.flush(session); lastBatchCount = batch.length; @@ -1211,12 +1224,14 @@ export abstract class MongoBucketBatch ); } - private groupSourceKeysByTable(sourceKeys: SourceKey[]): Map { - const grouped = new Map(); + private groupReplicaIdsByTable( + sourceKeys: { sourceTableId: bson.ObjectId; replicaId: storage.ReplicaId }[] + ): Map { + const grouped = new Map(); for (const sourceKey of sourceKeys) { - const existing = grouped.get(sourceKey.t) ?? []; - existing.push(sourceKey); - grouped.set(sourceKey.t, existing); + const existing = grouped.get(sourceKey.sourceTableId) ?? []; + existing.push(sourceKey.replicaId); + grouped.set(sourceKey.sourceTableId, existing); } return grouped; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts index ff259c028..a061c7e92 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts @@ -1,14 +1,18 @@ +import { mongo } from '@powersync/lib-service-mongodb'; import * as bson from 'bson'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; +import { idPrefixFilter } from '../../utils/util.js'; import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; +import { cacheKey } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; import { PersistedBatchV1 } from './PersistedBatchV1.js'; import { CommonCurrentBucket, CommonCurrentLookup, + CurrentDataDocumentId, CurrentDataDocument, SourceKey, isCurrentBucketV3, @@ -38,8 +42,12 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { return paramEvaluated.map((entry) => storage.serializeLookup(entry.lookup)); } + protected createCurrentDataId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId { + return { g: this.group_id, t: sourceTableId, k: replicaId } satisfies SourceKey; + } + protected createCurrentDataDocument( - id: SourceKey, + id: CurrentDataDocumentId, data: bson.Binary, buckets: CommonCurrentBucket[], lookups: CommonCurrentLookup[] @@ -58,12 +66,37 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { }); return { - _id: id, + _id: id as SourceKey, data, buckets: narrowedBuckets, lookups: narrowedLookups }; } + protected createCurrentDataLookupFilter(sourceTableId: bson.ObjectId, replicaIds: storage.ReplicaId[]) { + return { + _id: { + $in: replicaIds.map((replicaId) => this.createCurrentDataId(sourceTableId, replicaId) as SourceKey) + } + }; + } + + protected currentDataCacheKey(sourceTableId: bson.ObjectId, document: CurrentDataDocument): string { + return cacheKey(sourceTableId, document._id.k); + } + + protected currentDataReplicaId(document: CurrentDataDocument): storage.ReplicaId { + return document._id.k; + } + + protected activeCurrentDataFilter( + sourceTableId: bson.ObjectId + ): mongo.Filter { + return { + _id: idPrefixFilter({ g: this.group_id, t: sourceTableId }, ['k']), + pending_delete: { $exists: false } + } as mongo.Filter; + } + protected async cleanupCurrentData(_lastCheckpoint: bigint): Promise {} } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts index 4bfd22f3f..cbb53dfeb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts @@ -4,15 +4,16 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; +import { cacheKey } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; import { PersistedBatchV3 } from './PersistedBatchV3.js'; import { CommonCurrentBucket, CommonCurrentLookup, + CurrentDataDocumentId, CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, - SourceKey, isCurrentBucketV3, isRecordedLookupV3 } from './models.js'; @@ -47,8 +48,12 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { }); } + protected createCurrentDataId(_sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId { + return replicaId; + } + protected createCurrentDataDocument( - id: SourceKey, + id: CurrentDataDocumentId, data: bson.Binary, buckets: CommonCurrentBucket[], lookups: CommonCurrentLookup[] @@ -74,11 +79,30 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { }; } + protected createCurrentDataLookupFilter(_sourceTableId: bson.ObjectId, replicaIds: storage.ReplicaId[]) { + return { + _id: { $in: replicaIds } + }; + } + + protected currentDataCacheKey(sourceTableId: bson.ObjectId, document: CurrentDataDocumentV3): string { + return cacheKey(sourceTableId, document._id); + } + + protected currentDataReplicaId(document: CurrentDataDocumentV3): storage.ReplicaId { + return document._id; + } + + protected activeCurrentDataFilter(_sourceTableId: bson.ObjectId) { + return { + pending_delete: { $exists: false } + }; + } + protected async cleanupCurrentData(lastCheckpoint: bigint): Promise { let deletedCount = 0; for (const collection of await this.db.listCommonCurrentDataCollections(this.group_id)) { const result = await collection.deleteMany({ - '_id.g': this.group_id, pending_delete: { $exists: true, $lte: lastCheckpoint } }); deletedCount += result.deletedCount; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts index ce8da820e..cc9251ce4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts @@ -39,7 +39,7 @@ export class MongoParameterCompactor { } } - private async compactCollection(collection: mongo.Collection) { + private async compactCollection(collection: mongo.Collection) { // This is the currently-active checkpoint. // We do not remove any data that may be used by this checkpoint. // snapshot queries ensure that if any clients are still using older checkpoints, they would @@ -51,9 +51,11 @@ export class MongoParameterCompactor { // in MongoDB already. However, that risks running into cases where MongoDB needs to process // very large amounts of data before returning results, which could lead to timeouts. const cursor = collection.find( - { - 'key.g': this.group_id - }, + this.db.storageConfig.incrementalReprocessing + ? {} + : { + 'key.g': this.group_id + }, { sort: { lookup: 1, _id: 1 }, batchSize: 10_000, @@ -66,7 +68,7 @@ export class MongoParameterCompactor { max: this.options.compactParameterCacheLimit ?? 10_000 }); let removeIds: InternalOpId[] = []; - let removeDeleted: mongo.AnyBulkWriteOperation[] = []; + let removeDeleted: mongo.AnyBulkWriteOperation[] = []; let checkedEntries = 0; let checkedEntriesAtLastLog = 0; let lastProgressLogTime = Date.now(); @@ -121,7 +123,14 @@ export class MongoParameterCompactor { // in the cache due to cache size limits. So we need to explicitly remove all earlier operations. removeDeleted.push({ deleteMany: { - filter: { 'key.g': doc.key.g, lookup: doc.lookup, _id: { $lte: doc._id }, key: doc.key } + filter: this.db.storageConfig.incrementalReprocessing + ? { lookup: doc.lookup, _id: { $lte: doc._id }, key: doc.key } + : { + 'key.g': (doc.key as BucketParameterDocument['key']).g, + lookup: doc.lookup, + _id: { $lte: doc._id }, + key: doc.key + } } }); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index bfe8cec84..fde1815b4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -11,7 +11,7 @@ import { BucketStateDocument, CommonCurrentBucket, CommonCurrentLookup, - SourceKey, + CurrentDataDocumentId, TaggedBucketParameterDocument, TaggedBucketDataDocument } from './models.js'; @@ -56,7 +56,8 @@ export interface SaveParameterDataOptions { } export interface UpsertCurrentDataOptions { - id: SourceKey; + sourceTableId: bson.ObjectId; + id: CurrentDataDocumentId; data: bson.Binary | null; buckets: CommonCurrentBucket[]; lookups: CommonCurrentLookup[]; @@ -103,9 +104,13 @@ export abstract class PersistedBatch { abstract saveParameterData(data: SaveParameterDataOptions): void; - abstract hardDeleteCurrentData(id: SourceKey): void; + abstract hardDeleteCurrentData(sourceTableId: bson.ObjectId, id: CurrentDataDocumentId): void; - abstract softDeleteCurrentData(id: SourceKey, checkpointGreaterThan: bigint): void; + abstract softDeleteCurrentData( + sourceTableId: bson.ObjectId, + id: CurrentDataDocumentId, + checkpointGreaterThan: bigint + ): void; abstract upsertCurrentData(values: UpsertCurrentDataOptions): void; diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index 754633843..131cc1b24 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -150,7 +150,7 @@ export class PersistedBatchV1 extends PersistedBatch { } } - hardDeleteCurrentData(id: SourceKey) { + hardDeleteCurrentData(_sourceTableId: bson.ObjectId, id: SourceKey) { this.currentData.push({ deleteOne: { filter: { _id: id } @@ -159,8 +159,8 @@ export class PersistedBatchV1 extends PersistedBatch { this.currentSize += 50; } - softDeleteCurrentData(id: SourceKey, _checkpointGreaterThan: bigint) { - this.hardDeleteCurrentData(id); + softDeleteCurrentData(sourceTableId: bson.ObjectId, id: SourceKey, _checkpointGreaterThan: bigint) { + this.hardDeleteCurrentData(sourceTableId, id); } upsertCurrentData(values: UpsertCurrentDataOptions) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index dac4416ec..d218e7542 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -15,17 +15,18 @@ import { import { BucketParameterDocumentV3, CurrentBucketV3, + CurrentDataDocumentId, CurrentDataDocumentV3, isCurrentBucketV3, isRecordedLookupV3, RecordedLookupV3, - SourceKey, + SourceTableKey, taggedBucketParameterDocumentToV3, taggedBucketDataDocumentToV3 } from './models.js'; export class PersistedBatchV3 extends PersistedBatch { - currentData: mongo.AnyBulkWriteOperation[] = []; + currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation }[] = []; saveBucketData(options: SaveBucketDataOptions) { const remaining_buckets = new Map(); @@ -115,10 +116,9 @@ export class PersistedBatchV3 extends PersistedBatch { const values: BucketParameterDocumentV3 = { _id: op_id, key: { - g: this.group_id, t: mongoTableId(sourceTable.id), k: sourceKey - }, + } satisfies SourceTableKey, lookup: binLookup, bucket_parameters: result.bucketParameters }; @@ -136,10 +136,9 @@ export class PersistedBatchV3 extends PersistedBatch { const values: BucketParameterDocumentV3 = { _id: op_id, key: { - g: this.group_id, t: mongoTableId(sourceTable.id), k: sourceKey - }, + } satisfies SourceTableKey, lookup: lookup.l, bucket_parameters: [] }; @@ -152,28 +151,34 @@ export class PersistedBatchV3 extends PersistedBatch { } } - hardDeleteCurrentData(id: SourceKey) { + hardDeleteCurrentData(sourceTableId: bson.ObjectId, id: CurrentDataDocumentId) { this.currentData.push({ - deleteOne: { - filter: { _id: id } + sourceTableId, + operation: { + deleteOne: { + filter: { _id: id } + } } }); this.currentSize += 50; } - softDeleteCurrentData(id: SourceKey, checkpointGreaterThan: bigint) { + softDeleteCurrentData(sourceTableId: bson.ObjectId, id: CurrentDataDocumentId, checkpointGreaterThan: bigint) { this.currentData.push({ - updateOne: { - filter: { _id: id }, - update: { - $set: { - data: null, - buckets: [] as CurrentDataDocumentV3['buckets'], - lookups: [] as CurrentDataDocumentV3['lookups'], - pending_delete: checkpointGreaterThan - } - }, - upsert: true + sourceTableId, + operation: { + updateOne: { + filter: { _id: id }, + update: { + $set: { + data: null, + buckets: [] as CurrentDataDocumentV3['buckets'], + lookups: [] as CurrentDataDocumentV3['lookups'], + pending_delete: checkpointGreaterThan + } + }, + upsert: true + } } }); this.currentSize += 50; @@ -194,17 +199,20 @@ export class PersistedBatchV3 extends PersistedBatch { }); this.currentData.push({ - updateOne: { - filter: { _id: values.id }, - update: { - $set: { - data: values.data, - buckets, - lookups + sourceTableId: values.sourceTableId, + operation: { + updateOne: { + filter: { _id: values.id }, + update: { + $set: { + data: values.data, + buckets, + lookups + }, + $unset: { pending_delete: 1 } }, - $unset: { pending_delete: 1 } - }, - upsert: true + upsert: true + } } }); this.currentSize += (values.data?.length() ?? 0) + 100; @@ -263,44 +271,26 @@ export class PersistedBatchV3 extends PersistedBatch { protected async flushCurrentData(session: mongo.ClientSession) { const operationsBySourceTable = new Map(); for (const operation of this.currentData) { - const sourceTableId = this.getSourceTableIdHex(operation); - if (sourceTableId == null) { - throw new ReplicationAssertionError('Missing source table id for current_data operation'); - } + const sourceTableId = operation.sourceTableId.toHexString(); const existing = operationsBySourceTable.get(sourceTableId) ?? []; existing.push(operation); operationsBySourceTable.set(sourceTableId, existing); } for (const operations of operationsBySourceTable.values()) { - const firstOperation = operations[0]!; - const sourceTableId = this.getSourceTableId(firstOperation); - if (sourceTableId == null) { - throw new ReplicationAssertionError('Missing source table id for current_data bulkWrite'); - } + const sourceTableId = operations[0]!.sourceTableId; await this.db.initializeCurrentDataCollection(this.group_id, sourceTableId); - await this.db.v3_current_data(this.group_id, sourceTableId).bulkWrite(operations, { - session, - ordered: true - }); + await this.db.v3_current_data(this.group_id, sourceTableId).bulkWrite( + operations.map((entry) => entry.operation), + { + session, + ordered: true + } + ); } } protected resetCurrentData() { this.currentData = []; } - - private getSourceTableIdHex(operation: mongo.AnyBulkWriteOperation): string | undefined { - return this.getSourceTableId(operation)?.toHexString(); - } - - private getSourceTableId(operation: mongo.AnyBulkWriteOperation): bson.ObjectId | undefined { - if ('updateOne' in operation) { - return operation.updateOne.filter._id?.t; - } - if ('deleteOne' in operation) { - return operation.deleteOne.filter._id?.t; - } - return undefined; - } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index d881670b5..e0292b4a9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -295,7 +295,6 @@ export class PowerSyncMongo { await this.sourceRecords(replicationStreamId, sourceTableId).createIndex( { - '_id.g': 1, pending_delete: 1 }, { diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index ca74c7a25..8b0d5aaa8 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -23,6 +23,13 @@ export interface SourceKey { k: ReplicaId; } +export interface SourceTableKey { + /** source table id */ + t: bson.ObjectId; + /** source key */ + k: ReplicaId; +} + export interface BucketDataKeyV1 { /** group_id */ g: number; @@ -56,7 +63,7 @@ export interface RecordedLookupV3 { } export interface CurrentDataDocumentV3 { - _id: SourceKey; + _id: ReplicaId; data: bson.Binary | null; buckets: CurrentBucketV3[]; lookups: RecordedLookupV3[]; @@ -81,9 +88,15 @@ export interface BucketParameterDocument { bucket_parameters: Record[]; } -export interface BucketParameterDocumentV3 extends BucketParameterDocument {} +export interface BucketParameterDocumentV3 extends Omit { + key: SourceTableKey; +} -export interface TaggedBucketParameterDocument extends BucketParameterDocumentV3 { +export interface TaggedBucketParameterDocument { + _id: bigint; + key: BucketParameterDocument['key'] | BucketParameterDocumentV3['key']; + lookup: bson.Binary; + bucket_parameters: Record[]; index: ParameterIndexId; } @@ -170,12 +183,12 @@ export function bucketParameterDocumentToTagged( export function taggedBucketParameterDocumentToV1(document: TaggedBucketParameterDocument): BucketParameterDocument { const { index: _index, ...rest } = document; - return rest; + return rest as BucketParameterDocument; } export function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3 { const { index: _index, ...rest } = document; - return rest; + return rest as BucketParameterDocumentV3; } export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; @@ -407,6 +420,7 @@ export interface InstanceDocument { export interface ClientConnectionDocument extends event_types.ClientConnection {} +export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id']; export type CommonCurrentDataDocument = CurrentDataDocument | CurrentDataDocumentV3; export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3; export type CommonCurrentLookup = bson.Binary | RecordedLookupV3; From e0b5d397d18d065bf03881e5fa7671e71745d4c0 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 11:55:52 +0200 Subject: [PATCH 26/93] Split out CurrentDataStore. --- .../implementation/CurrentDataStore.ts | 43 ++++++ .../implementation/CurrentDataStoreV1.ts | 140 ++++++++++++++++++ .../implementation/CurrentDataStoreV3.ts | 139 +++++++++++++++++ .../implementation/MongoBucketBatch.ts | 130 +++------------- .../implementation/MongoBucketBatchV1.ts | 76 +--------- .../implementation/MongoBucketBatchV3.ts | 84 +---------- 6 files changed, 362 insertions(+), 250 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts new file mode 100644 index 000000000..cfff04884 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts @@ -0,0 +1,43 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { Logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import * as bson from 'bson'; +import { CommonCurrentBucket, CommonCurrentLookup, CurrentDataDocumentId } from './models.js'; + +export interface CurrentDataLookupEntry { + sourceTableId: bson.ObjectId; + replicaId: storage.ReplicaId; +} + +export interface LoadedCurrentData { + sourceTableId: bson.ObjectId; + id: CurrentDataDocumentId; + replicaId: storage.ReplicaId; + data: bson.Binary | null; + buckets: CommonCurrentBucket[]; + lookups: CommonCurrentLookup[]; + cacheKey: string; +} + +export interface CurrentDataStore { + createId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId; + createLoadedDocument( + sourceTableId: bson.ObjectId, + id: CurrentDataDocumentId, + data: bson.Binary | null, + buckets: CommonCurrentBucket[], + lookups: CommonCurrentLookup[] + ): LoadedCurrentData; + loadSizes(session: mongo.ClientSession, entries: CurrentDataLookupEntry[]): Promise>; + loadDocuments( + session: mongo.ClientSession, + entries: CurrentDataLookupEntry[], + idsOnly: boolean + ): Promise>; + loadTruncateBatch( + session: mongo.ClientSession, + sourceTableId: bson.ObjectId, + limit: number + ): Promise; + cleanup(lastCheckpoint: bigint, logger: Logger): Promise; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts new file mode 100644 index 000000000..29fd7ff37 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts @@ -0,0 +1,140 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { Logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import * as bson from 'bson'; +import { idPrefixFilter } from '../../utils/util.js'; +import { VersionedPowerSyncMongo } from './db.js'; +import { cacheKey } from './OperationBatch.js'; +import { CurrentDataStore, CurrentDataLookupEntry, LoadedCurrentData } from './CurrentDataStore.js'; +import { CurrentDataDocument, CurrentDataDocumentId, SourceKey } from './models.js'; + +export class CurrentDataStoreV1 implements CurrentDataStore { + constructor( + private readonly db: VersionedPowerSyncMongo, + private readonly groupId: number + ) {} + + createId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId { + return { + g: this.groupId, + t: sourceTableId, + k: replicaId + } satisfies SourceKey; + } + + createLoadedDocument( + sourceTableId: bson.ObjectId, + id: CurrentDataDocumentId, + data: bson.Binary | null, + buckets: LoadedCurrentData['buckets'], + lookups: LoadedCurrentData['lookups'] + ): LoadedCurrentData { + const typedId = id as SourceKey; + return { + sourceTableId, + id: typedId, + replicaId: typedId.k, + data, + buckets, + lookups, + cacheKey: cacheKey(sourceTableId, typedId.k) + }; + } + + async loadSizes(session: mongo.ClientSession, entries: CurrentDataLookupEntry[]): Promise> { + const sizes = new Map(); + for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { + const sizeCursor: mongo.AggregationCursor = this.db + .v1_current_data(this.groupId, sourceTableId) + .aggregate( + [ + { + $match: { + _id: { + $in: replicaIds.map((replicaId) => this.createId(sourceTableId, replicaId) as SourceKey) + } + } + }, + { + $project: { + _id: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ], + { session } + ); + for await (const doc of sizeCursor.stream()) { + sizes.set(cacheKey(sourceTableId, doc._id.k), doc.size); + } + } + return sizes; + } + + async loadDocuments( + session: mongo.ClientSession, + entries: CurrentDataLookupEntry[], + idsOnly: boolean + ): Promise> { + const documents = new Map(); + const projection = idsOnly ? { _id: 1 } : undefined; + for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { + const cursor = this.db.v1_current_data(this.groupId, sourceTableId).find( + { + _id: { + $in: replicaIds.map((replicaId) => this.createId(sourceTableId, replicaId) as SourceKey) + } + }, + { session, projection } + ); + for await (const doc of cursor.stream()) { + const loaded = this.createLoadedDocument( + sourceTableId, + doc._id, + idsOnly ? null : doc.data, + idsOnly ? [] : doc.buckets, + idsOnly ? [] : doc.lookups + ); + documents.set(loaded.cacheKey, loaded); + } + } + return documents; + } + + async loadTruncateBatch( + session: mongo.ClientSession, + sourceTableId: bson.ObjectId, + limit: number + ): Promise { + const cursor = this.db.v1_current_data(this.groupId, sourceTableId).find( + { + _id: idPrefixFilter({ g: this.groupId, t: sourceTableId }, ['k']), + pending_delete: { $exists: false } + }, + { + projection: { + _id: 1, + buckets: 1, + lookups: 1 + }, + limit, + session + } + ); + return (await cursor.toArray()).map((doc) => + this.createLoadedDocument(sourceTableId, doc._id, null, doc.buckets, doc.lookups) + ); + } + + async cleanup(_lastCheckpoint: bigint, _logger: Logger): Promise {} + + private groupEntries(entries: CurrentDataLookupEntry[]): Map { + const grouped = new Map(); + for (const entry of entries) { + const existing = grouped.get(entry.sourceTableId) ?? []; + existing.push(entry.replicaId); + grouped.set(entry.sourceTableId, existing); + } + return grouped; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts new file mode 100644 index 000000000..804b9fcbf --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts @@ -0,0 +1,139 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { Logger } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import * as bson from 'bson'; +import { VersionedPowerSyncMongo } from './db.js'; +import { cacheKey } from './OperationBatch.js'; +import { CurrentDataStore, CurrentDataLookupEntry, LoadedCurrentData } from './CurrentDataStore.js'; +import { CurrentDataDocumentV3, CurrentDataDocumentId } from './models.js'; + +export class CurrentDataStoreV3 implements CurrentDataStore { + constructor( + private readonly db: VersionedPowerSyncMongo, + private readonly groupId: number + ) {} + + createId(_sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId { + return replicaId; + } + + createLoadedDocument( + sourceTableId: bson.ObjectId, + id: CurrentDataDocumentId, + data: bson.Binary | null, + buckets: LoadedCurrentData['buckets'], + lookups: LoadedCurrentData['lookups'] + ): LoadedCurrentData { + return { + sourceTableId, + id, + replicaId: id, + data, + buckets, + lookups, + cacheKey: cacheKey(sourceTableId, id) + }; + } + + async loadSizes(session: mongo.ClientSession, entries: CurrentDataLookupEntry[]): Promise> { + const sizes = new Map(); + for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { + const filter = { + _id: { $in: replicaIds as any[] } + } as unknown as mongo.Filter; + const sizeCursor: mongo.AggregationCursor = this.db + .v3_current_data(this.groupId, sourceTableId) + .aggregate( + [ + { + $match: filter + }, + { + $project: { + _id: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ], + { session } + ); + for await (const doc of sizeCursor.stream()) { + sizes.set(cacheKey(sourceTableId, doc._id), doc.size); + } + } + return sizes; + } + + async loadDocuments( + session: mongo.ClientSession, + entries: CurrentDataLookupEntry[], + idsOnly: boolean + ): Promise> { + const documents = new Map(); + const projection = idsOnly ? { _id: 1 } : undefined; + for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { + const filter = { + _id: { $in: replicaIds as any[] } + } as unknown as mongo.Filter; + const cursor = this.db.v3_current_data(this.groupId, sourceTableId).find(filter, { session, projection }); + for await (const doc of cursor.stream()) { + const loaded = this.createLoadedDocument( + sourceTableId, + doc._id, + idsOnly ? null : doc.data, + idsOnly ? [] : doc.buckets, + idsOnly ? [] : doc.lookups + ); + documents.set(loaded.cacheKey, loaded); + } + } + return documents; + } + + async loadTruncateBatch( + session: mongo.ClientSession, + sourceTableId: bson.ObjectId, + limit: number + ): Promise { + const cursor = this.db.v3_current_data(this.groupId, sourceTableId).find( + { + pending_delete: { $exists: false } + }, + { + projection: { + _id: 1, + buckets: 1, + lookups: 1 + }, + limit, + session + } + ); + return (await cursor.toArray()).map((doc) => + this.createLoadedDocument(sourceTableId, doc._id, null, doc.buckets, doc.lookups) + ); + } + + async cleanup(lastCheckpoint: bigint, logger: Logger): Promise { + let deletedCount = 0; + for (const collection of await this.db.listCommonCurrentDataCollections(this.groupId)) { + const result = await collection.deleteMany({ + pending_delete: { $exists: true, $lte: lastCheckpoint } + }); + deletedCount += result.deletedCount; + } + if (deletedCount > 0) { + logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`); + } + } + + private groupEntries(entries: CurrentDataLookupEntry[]): Map { + const grouped = new Map(); + for (const entry of entries) { + const existing = grouped.get(entry.sourceTableId) ?? []; + existing.push(entry.replicaId); + grouped.set(entry.sourceTableId, existing); + } + return grouped; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 04dfb5011..8c2a77ccf 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -31,19 +31,13 @@ import { utils } from '@powersync/service-core'; import * as timers from 'node:timers/promises'; -import { idPrefixFilter, mongoTableId } from '../../utils/util.js'; +import { mongoTableId } from '../../utils/util.js'; import { VersionedPowerSyncMongo } from './db.js'; -import { - CommonCurrentBucket, - CommonCurrentLookup, - CommonCurrentDataDocument, - CurrentDataDocumentId, - SourceKey, - SyncRuleDocument -} from './models.js'; +import { CommonCurrentBucket, CommonCurrentLookup, SyncRuleDocument } from './models.js'; +import { CurrentDataStore, LoadedCurrentData } from './CurrentDataStore.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; -import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js'; +import { OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; @@ -165,30 +159,7 @@ export abstract class MongoBucketBatch protected abstract mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CommonCurrentLookup[]; - protected abstract createCurrentDataId( - sourceTableId: bson.ObjectId, - replicaId: storage.ReplicaId - ): CurrentDataDocumentId; - - protected abstract createCurrentDataDocument( - id: CurrentDataDocumentId, - data: bson.Binary, - buckets: CommonCurrentBucket[], - lookups: CommonCurrentLookup[] - ): CommonCurrentDataDocument; - - protected abstract createCurrentDataLookupFilter( - sourceTableId: bson.ObjectId, - replicaIds: storage.ReplicaId[] - ): mongo.Filter; - - protected abstract currentDataCacheKey(sourceTableId: bson.ObjectId, document: CommonCurrentDataDocument): string; - - protected abstract currentDataReplicaId(document: CommonCurrentDataDocument): storage.ReplicaId; - - protected abstract activeCurrentDataFilter(sourceTableId: bson.ObjectId): mongo.Filter; - - protected abstract cleanupCurrentData(lastCheckpoint: bigint): Promise; + protected abstract get currentDataStore(): CurrentDataStore; async flush(options?: storage.BatchBucketFlushOptions): Promise { let result: storage.FlushedResult | null = null; @@ -260,32 +231,7 @@ export abstract class MongoBucketBatch replicaId: r.beforeId })); - sizes = new Map(); - - for (const [sourceTableId, replicaIds] of this.groupReplicaIdsByTable(sizeLookups)) { - const sizeCursor: mongo.AggregationCursor = this.db - .common_current_data(this.group_id, sourceTableId) - .aggregate( - [ - { - $match: { - ...this.createCurrentDataLookupFilter(sourceTableId, replicaIds) - } - }, - { - $project: { - _id: 1, - size: { $bsonSize: '$$ROOT' } - } - } - ], - { session } - ); - for await (let doc of sizeCursor.stream()) { - const key = this.currentDataCacheKey(sourceTableId, doc); - sizes.set(key, doc.size); - } - } + sizes = await this.currentDataStore.loadSizes(session, sizeLookups); } // If set, we need to start a new transaction with this batch. @@ -307,17 +253,7 @@ export abstract class MongoBucketBatch sourceTableId: mongoTableId(r.record.sourceTable.id), replicaId: r.beforeId })); - let current_data_lookup = new Map(); - // With skipExistingRows, we only need to know whether or not the row exists. - const projection = this.skipExistingRows ? { _id: 1 } : undefined; - for (const [sourceTableId, replicaIds] of this.groupReplicaIdsByTable(lookups)) { - const cursor = this.db - .common_current_data(this.group_id, sourceTableId) - .find(this.createCurrentDataLookupFilter(sourceTableId, replicaIds), { session, projection }); - for await (let doc of cursor.stream()) { - current_data_lookup.set(this.currentDataCacheKey(sourceTableId, doc), doc); - } - } + let current_data_lookup = await this.currentDataStore.loadDocuments(session, lookups, this.skipExistingRows); let persistedBatch: PersistedBatch | null = this.createPersistedBatch(transactionSize); @@ -368,7 +304,7 @@ export abstract class MongoBucketBatch private saveOperation( batch: PersistedBatch, operation: RecordOperation, - current_data: CommonCurrentDataDocument | null, + current_data: LoadedCurrentData | null, opSeq: MongoIdSequence ) { const record = operation.record; @@ -383,7 +319,7 @@ export abstract class MongoBucketBatch let new_lookups: CommonCurrentLookup[] = []; const sourceTableId = mongoTableId(record.sourceTable.id); - const before_key = this.createCurrentDataId(sourceTableId, beforeId); + const before_key = this.currentDataStore.createId(sourceTableId, beforeId); if (this.skipExistingRows) { if (record.tag == SaveOperationTag.INSERT) { @@ -581,12 +517,12 @@ export abstract class MongoBucketBatch } } - let result: CommonCurrentDataDocument | null = null; + let result: LoadedCurrentData | null = null; // 5. TOAST: Update current data and bucket list. if (afterId) { // Insert or update - const after_key = this.createCurrentDataId(sourceTableId, afterId); + const after_key = this.currentDataStore.createId(sourceTableId, afterId); batch.upsertCurrentData({ sourceTableId, id: after_key, @@ -594,7 +530,13 @@ export abstract class MongoBucketBatch buckets: new_buckets, lookups: new_lookups }); - result = this.createCurrentDataDocument(after_key, afterData!, new_buckets, new_lookups); + result = this.currentDataStore.createLoadedDocument( + sourceTableId, + after_key, + afterData, + new_buckets, + new_lookups + ); } if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { @@ -884,7 +826,7 @@ export abstract class MongoBucketBatch this.persisted_op = null; this.last_checkpoint_lsn = lsn; if (newLastCheckpoint != null) { - await this.cleanupCurrentData(newLastCheckpoint); + await this.currentDataStore.cleanup(newLastCheckpoint, this.logger); } } return { checkpointBlocked, checkpointCreated }; @@ -1045,20 +987,8 @@ export abstract class MongoBucketBatch let lastBatchCount = BATCH_LIMIT; while (lastBatchCount == BATCH_LIMIT) { await this.withReplicationTransaction(`Truncate ${sourceTable.qualifiedName}`, async (session, opSeq) => { - const current_data_filter = this.activeCurrentDataFilter(mongoTableId(sourceTable.id)); - - const cursor = this.db - .common_current_data(this.group_id, mongoTableId(sourceTable.id)) - .find(current_data_filter, { - projection: { - _id: 1, - buckets: 1, - lookups: 1 - }, - limit: BATCH_LIMIT, - session: session - }); - const batch = await cursor.toArray(); + const sourceTableId = mongoTableId(sourceTable.id); + const batch = await this.currentDataStore.loadTruncateBatch(session, sourceTableId, BATCH_LIMIT); const persistedBatch = this.createPersistedBatch(0); for (let value of batch) { @@ -1067,18 +997,18 @@ export abstract class MongoBucketBatch before_buckets: value.buckets, evaluated: [], table: sourceTable, - sourceKey: this.currentDataReplicaId(value) + sourceKey: value.replicaId }); persistedBatch.saveParameterData({ op_seq: opSeq, existing_lookups: value.lookups, evaluated: [], sourceTable: sourceTable, - sourceKey: this.currentDataReplicaId(value) + sourceKey: value.replicaId }); // Since this is not from streaming replication, we can do a hard delete - persistedBatch.hardDeleteCurrentData(mongoTableId(sourceTable.id), value._id); + persistedBatch.hardDeleteCurrentData(sourceTableId, value.id); } await persistedBatch.flush(session); lastBatchCount = batch.length; @@ -1223,16 +1153,4 @@ export abstract class MongoBucketBatch [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table)) ); } - - private groupReplicaIdsByTable( - sourceKeys: { sourceTableId: bson.ObjectId; replicaId: storage.ReplicaId }[] - ): Map { - const grouped = new Map(); - for (const sourceKey of sourceKeys) { - const existing = grouped.get(sourceKey.sourceTableId) ?? []; - existing.push(sourceKey.replicaId); - grouped.set(sourceKey.sourceTableId, existing); - } - return grouped; - } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts index a061c7e92..5d05c63f8 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts @@ -1,27 +1,21 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import * as bson from 'bson'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; -import { idPrefixFilter } from '../../utils/util.js'; import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; -import { cacheKey } from './OperationBatch.js'; +import { CurrentDataStore } from './CurrentDataStore.js'; +import { CurrentDataStoreV1 } from './CurrentDataStoreV1.js'; import { PersistedBatch } from './PersistedBatch.js'; import { PersistedBatchV1 } from './PersistedBatchV1.js'; -import { - CommonCurrentBucket, - CommonCurrentLookup, - CurrentDataDocumentId, - CurrentDataDocument, - SourceKey, - isCurrentBucketV3, - isRecordedLookupV3 -} from './models.js'; +import { CommonCurrentBucket, CommonCurrentLookup, isCurrentBucketV3, isRecordedLookupV3 } from './models.js'; export class MongoBucketBatchV1 extends MongoBucketBatch { + private readonly store: CurrentDataStore; + constructor(options: MongoBucketBatchOptions) { super(options); + this.store = new CurrentDataStoreV1(this.db, this.group_id); } protected createPersistedBatch(writtenSize: number): PersistedBatch { @@ -42,61 +36,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { return paramEvaluated.map((entry) => storage.serializeLookup(entry.lookup)); } - protected createCurrentDataId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId { - return { g: this.group_id, t: sourceTableId, k: replicaId } satisfies SourceKey; - } - - protected createCurrentDataDocument( - id: CurrentDataDocumentId, - data: bson.Binary, - buckets: CommonCurrentBucket[], - lookups: CommonCurrentLookup[] - ): CurrentDataDocument { - const narrowedBuckets = buckets.map((bucket) => { - if (isCurrentBucketV3(bucket)) { - throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); - } - return bucket; - }); - const narrowedLookups = lookups.map((lookup) => { - if (isRecordedLookupV3(lookup)) { - throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); - } - return lookup; - }); - - return { - _id: id as SourceKey, - data, - buckets: narrowedBuckets, - lookups: narrowedLookups - }; - } - - protected createCurrentDataLookupFilter(sourceTableId: bson.ObjectId, replicaIds: storage.ReplicaId[]) { - return { - _id: { - $in: replicaIds.map((replicaId) => this.createCurrentDataId(sourceTableId, replicaId) as SourceKey) - } - }; + protected get currentDataStore(): CurrentDataStore { + return this.store; } - - protected currentDataCacheKey(sourceTableId: bson.ObjectId, document: CurrentDataDocument): string { - return cacheKey(sourceTableId, document._id.k); - } - - protected currentDataReplicaId(document: CurrentDataDocument): storage.ReplicaId { - return document._id.k; - } - - protected activeCurrentDataFilter( - sourceTableId: bson.ObjectId - ): mongo.Filter { - return { - _id: idPrefixFilter({ g: this.group_id, t: sourceTableId }, ['k']), - pending_delete: { $exists: false } - } as mongo.Filter; - } - - protected async cleanupCurrentData(_lastCheckpoint: bigint): Promise {} } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts index cbb53dfeb..5c4ad6198 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts @@ -1,26 +1,20 @@ import * as bson from 'bson'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; -import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; -import { cacheKey } from './OperationBatch.js'; +import { CurrentDataStore } from './CurrentDataStore.js'; +import { CurrentDataStoreV3 } from './CurrentDataStoreV3.js'; import { PersistedBatch } from './PersistedBatch.js'; import { PersistedBatchV3 } from './PersistedBatchV3.js'; -import { - CommonCurrentBucket, - CommonCurrentLookup, - CurrentDataDocumentId, - CurrentBucketV3, - CurrentDataDocumentV3, - RecordedLookupV3, - isCurrentBucketV3, - isRecordedLookupV3 -} from './models.js'; +import { CommonCurrentBucket, CommonCurrentLookup, CurrentBucketV3, RecordedLookupV3 } from './models.js'; export class MongoBucketBatchV3 extends MongoBucketBatch { + private readonly store: CurrentDataStore; + constructor(options: MongoBucketBatchOptions) { super(options); + this.store = new CurrentDataStoreV3(this.db, this.group_id); } protected createPersistedBatch(writtenSize: number): PersistedBatch { @@ -48,69 +42,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { }); } - protected createCurrentDataId(_sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId { - return replicaId; - } - - protected createCurrentDataDocument( - id: CurrentDataDocumentId, - data: bson.Binary, - buckets: CommonCurrentBucket[], - lookups: CommonCurrentLookup[] - ): CurrentDataDocumentV3 { - const narrowedBuckets = buckets.map((bucket) => { - if (!isCurrentBucketV3(bucket)) { - throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); - } - return bucket; - }); - const narrowedLookups = lookups.map((lookup) => { - if (!isRecordedLookupV3(lookup)) { - throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); - } - return lookup; - }); - - return { - _id: id, - data, - buckets: narrowedBuckets, - lookups: narrowedLookups - }; - } - - protected createCurrentDataLookupFilter(_sourceTableId: bson.ObjectId, replicaIds: storage.ReplicaId[]) { - return { - _id: { $in: replicaIds } - }; - } - - protected currentDataCacheKey(sourceTableId: bson.ObjectId, document: CurrentDataDocumentV3): string { - return cacheKey(sourceTableId, document._id); - } - - protected currentDataReplicaId(document: CurrentDataDocumentV3): storage.ReplicaId { - return document._id; - } - - protected activeCurrentDataFilter(_sourceTableId: bson.ObjectId) { - return { - pending_delete: { $exists: false } - }; - } - - protected async cleanupCurrentData(lastCheckpoint: bigint): Promise { - let deletedCount = 0; - for (const collection of await this.db.listCommonCurrentDataCollections(this.group_id)) { - const result = await collection.deleteMany({ - pending_delete: { $exists: true, $lte: lastCheckpoint } - }); - deletedCount += result.deletedCount; - } - if (deletedCount > 0) { - this.logger.info( - `Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}` - ); - } + protected get currentDataStore(): CurrentDataStore { + return this.store; } } From a2c46363db2ed33013ac690c6113cba8a3e17d4e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 12:10:50 +0200 Subject: [PATCH 27/93] Further split out implementations. --- .../implementation/CurrentDataStore.ts | 30 ++++++---- .../implementation/CurrentDataStoreV1.ts | 54 +++++++++++++----- .../implementation/CurrentDataStoreV3.ts | 44 ++++++++++---- .../implementation/MongoBucketBatch.ts | 50 +++++++--------- .../implementation/MongoBucketBatchShared.ts | 6 +- .../implementation/MongoBucketBatchV1.ts | 18 ------ .../implementation/MongoBucketBatchV3.ts | 26 +-------- .../storage/implementation/PersistedBatch.ts | 24 +++----- .../implementation/PersistedBatchV1.ts | 44 ++++++++------ .../implementation/PersistedBatchV3.ts | 57 +++++++++++-------- 10 files changed, 185 insertions(+), 168 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts index cfff04884..b6eca0460 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts @@ -1,33 +1,39 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; +import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { CommonCurrentBucket, CommonCurrentLookup, CurrentDataDocumentId } from './models.js'; +import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; export interface CurrentDataLookupEntry { sourceTableId: bson.ObjectId; replicaId: storage.ReplicaId; } +export interface CurrentDataBucketState { + definitionId: BucketDefinitionId | null; + bucket: string; + table: string; + id: string; +} + +export interface CurrentDataLookupState { + indexId: ParameterIndexId | null; + lookup: bson.Binary; +} + export interface LoadedCurrentData { sourceTableId: bson.ObjectId; - id: CurrentDataDocumentId; replicaId: storage.ReplicaId; data: bson.Binary | null; - buckets: CommonCurrentBucket[]; - lookups: CommonCurrentLookup[]; + buckets: CurrentDataBucketState[]; + lookups: CurrentDataLookupState[]; cacheKey: string; } export interface CurrentDataStore { - createId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId; - createLoadedDocument( - sourceTableId: bson.ObjectId, - id: CurrentDataDocumentId, - data: bson.Binary | null, - buckets: CommonCurrentBucket[], - lookups: CommonCurrentLookup[] - ): LoadedCurrentData; + mapEvaluatedBuckets(evaluated: EvaluatedRow[]): CurrentDataBucketState[]; + mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CurrentDataLookupState[]; loadSizes(session: mongo.ClientSession, entries: CurrentDataLookupEntry[]): Promise>; loadDocuments( session: mongo.ClientSession, diff --git a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts index 29fd7ff37..39913fd55 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts @@ -5,8 +5,14 @@ import * as bson from 'bson'; import { idPrefixFilter } from '../../utils/util.js'; import { VersionedPowerSyncMongo } from './db.js'; import { cacheKey } from './OperationBatch.js'; -import { CurrentDataStore, CurrentDataLookupEntry, LoadedCurrentData } from './CurrentDataStore.js'; -import { CurrentDataDocument, CurrentDataDocumentId, SourceKey } from './models.js'; +import { + CurrentDataStore, + CurrentDataLookupEntry, + CurrentDataLookupState, + LoadedCurrentData +} from './CurrentDataStore.js'; +import { CurrentDataDocument, SourceKey } from './models.js'; +import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; export class CurrentDataStoreV1 implements CurrentDataStore { constructor( @@ -14,7 +20,23 @@ export class CurrentDataStoreV1 implements CurrentDataStore { private readonly groupId: number ) {} - createId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId { + mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedCurrentData['buckets'] { + return evaluated.map((entry) => ({ + definitionId: null, + bucket: entry.bucket, + table: entry.table, + id: entry.id + })); + } + + mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CurrentDataLookupState[] { + return paramEvaluated.map((entry) => ({ + indexId: null, + lookup: storage.serializeLookup(entry.lookup) + })); + } + + private createId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey { return { g: this.groupId, t: sourceTableId, @@ -22,22 +44,28 @@ export class CurrentDataStoreV1 implements CurrentDataStore { } satisfies SourceKey; } - createLoadedDocument( + private createLoadedDocument( sourceTableId: bson.ObjectId, - id: CurrentDataDocumentId, + id: SourceKey, data: bson.Binary | null, - buckets: LoadedCurrentData['buckets'], - lookups: LoadedCurrentData['lookups'] + buckets: CurrentDataDocument['buckets'], + lookups: CurrentDataDocument['lookups'] ): LoadedCurrentData { - const typedId = id as SourceKey; return { sourceTableId, - id: typedId, - replicaId: typedId.k, + replicaId: id.k, data, - buckets, - lookups, - cacheKey: cacheKey(sourceTableId, typedId.k) + buckets: buckets.map((bucket) => ({ + definitionId: null, + bucket: bucket.bucket, + table: bucket.table, + id: bucket.id + })), + lookups: lookups.map((lookup) => ({ + indexId: null, + lookup + })), + cacheKey: cacheKey(sourceTableId, id.k) }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts index 804b9fcbf..7098e1b93 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts @@ -2,35 +2,57 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import * as bson from 'bson'; +import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import { VersionedPowerSyncMongo } from './db.js'; import { cacheKey } from './OperationBatch.js'; import { CurrentDataStore, CurrentDataLookupEntry, LoadedCurrentData } from './CurrentDataStore.js'; -import { CurrentDataDocumentV3, CurrentDataDocumentId } from './models.js'; +import { CurrentDataDocumentV3 } from './models.js'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; export class CurrentDataStoreV3 implements CurrentDataStore { constructor( private readonly db: VersionedPowerSyncMongo, - private readonly groupId: number + private readonly groupId: number, + private readonly mapping: BucketDefinitionMapping ) {} - createId(_sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): CurrentDataDocumentId { - return replicaId; + mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedCurrentData['buckets'] { + return evaluated.map((entry) => ({ + definitionId: this.mapping.bucketSourceId(entry.source), + bucket: entry.bucket, + table: entry.table, + id: entry.id + })); } - createLoadedDocument( + mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedCurrentData['lookups'] { + return paramEvaluated.map((entry) => ({ + indexId: this.mapping.parameterLookupId(entry.lookup.source), + lookup: storage.serializeLookup(entry.lookup) + })); + } + + private createLoadedDocument( sourceTableId: bson.ObjectId, - id: CurrentDataDocumentId, + id: storage.ReplicaId, data: bson.Binary | null, - buckets: LoadedCurrentData['buckets'], - lookups: LoadedCurrentData['lookups'] + buckets: CurrentDataDocumentV3['buckets'], + lookups: CurrentDataDocumentV3['lookups'] ): LoadedCurrentData { return { sourceTableId, - id, replicaId: id, data, - buckets, - lookups, + buckets: buckets.map((bucket) => ({ + definitionId: bucket.def, + bucket: bucket.bucket, + table: bucket.table, + id: bucket.id + })), + lookups: lookups.map((lookup) => ({ + indexId: lookup.i, + lookup: lookup.l + })), cacheKey: cacheKey(sourceTableId, id) }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 8c2a77ccf..05849ae9a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -1,12 +1,5 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { - EvaluatedParameters, - EvaluatedRow, - HydratedSyncRules, - SqlEventDescriptor, - SqliteRow, - SqliteValue -} from '@powersync/service-sync-rules'; +import { HydratedSyncRules, SqlEventDescriptor, SqliteRow, SqliteValue } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { @@ -33,14 +26,14 @@ import { import * as timers from 'node:timers/promises'; import { mongoTableId } from '../../utils/util.js'; import { VersionedPowerSyncMongo } from './db.js'; -import { CommonCurrentBucket, CommonCurrentLookup, SyncRuleDocument } from './models.js'; +import { SyncRuleDocument } from './models.js'; import { CurrentDataStore, LoadedCurrentData } from './CurrentDataStore.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { OperationBatch, RecordOperation } from './OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; +import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; // Currently, we can only have a single flush() at a time, since it locks the op_id sequence. // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex @@ -155,10 +148,6 @@ export abstract class MongoBucketBatch protected abstract createPersistedBatch(writtenSize: number): PersistedBatch; - protected abstract mapEvaluatedBuckets(evaluated: EvaluatedRow[]): CommonCurrentBucket[]; - - protected abstract mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CommonCurrentLookup[]; - protected abstract get currentDataStore(): CurrentDataStore; async flush(options?: storage.BatchBucketFlushOptions): Promise { @@ -313,13 +302,12 @@ export abstract class MongoBucketBatch let after = record.after; const sourceTable = record.sourceTable; - let existing_buckets: CommonCurrentBucket[] = []; - let new_buckets: CommonCurrentBucket[] = []; - let existing_lookups: CommonCurrentLookup[] = []; - let new_lookups: CommonCurrentLookup[] = []; + let existing_buckets: LoadedCurrentData['buckets'] = []; + let new_buckets: LoadedCurrentData['buckets'] = []; + let existing_lookups: LoadedCurrentData['lookups'] = []; + let new_lookups: LoadedCurrentData['lookups'] = []; const sourceTableId = mongoTableId(record.sourceTable.id); - const before_key = this.currentDataStore.createId(sourceTableId, beforeId); if (this.skipExistingRows) { if (record.tag == SaveOperationTag.INSERT) { @@ -480,7 +468,7 @@ export abstract class MongoBucketBatch table: sourceTable, before_buckets: existing_buckets }); - new_buckets = this.mapEvaluatedBuckets(evaluated); + new_buckets = this.currentDataStore.mapEvaluatedBuckets(evaluated); } if (sourceTable.syncParameters) { @@ -513,7 +501,7 @@ export abstract class MongoBucketBatch evaluated: paramEvaluated, existing_lookups }); - new_lookups = this.mapParameterLookups(paramEvaluated); + new_lookups = this.currentDataStore.mapParameterLookups(paramEvaluated); } } @@ -522,21 +510,21 @@ export abstract class MongoBucketBatch // 5. TOAST: Update current data and bucket list. if (afterId) { // Insert or update - const after_key = this.currentDataStore.createId(sourceTableId, afterId); batch.upsertCurrentData({ sourceTableId, - id: after_key, + replicaId: afterId, data: afterData, buckets: new_buckets, lookups: new_lookups }); - result = this.currentDataStore.createLoadedDocument( + result = { sourceTableId, - after_key, - afterData, - new_buckets, - new_lookups - ); + replicaId: afterId, + data: afterData, + buckets: new_buckets, + lookups: new_lookups, + cacheKey: operation.internalAfterKey! + }; } if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { @@ -544,7 +532,7 @@ export abstract class MongoBucketBatch // Note that this is a soft delete. // We don't specifically need a new or unique op_id here, but it must be greater than the // last checkpoint, so we use next(). - batch.softDeleteCurrentData(sourceTableId, before_key, opSeq.next()); + batch.softDeleteCurrentData(sourceTableId, beforeId, opSeq.next()); } return result; } @@ -1008,7 +996,7 @@ export abstract class MongoBucketBatch }); // Since this is not from streaming replication, we can do a hard delete - persistedBatch.hardDeleteCurrentData(sourceTableId, value.id); + persistedBatch.hardDeleteCurrentData(sourceTableId, value.replicaId); } await persistedBatch.flush(session); lastBatchCount = batch.length; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts index 1477b8711..ac15cf459 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts @@ -1,11 +1,11 @@ import * as bson from 'bson'; -import { CommonCurrentBucket } from './models.js'; +import { CurrentDataBucketState } from './CurrentDataStore.js'; export const MAX_ROW_SIZE = 15 * 1024 * 1024; export const EMPTY_DATA = new bson.Binary(bson.serialize({})); -export function currentBucketKey(bucket: CommonCurrentBucket) { - const prefix = 'def' in bucket ? `${bucket.def}:` : ''; +export function currentBucketKey(bucket: CurrentDataBucketState) { + const prefix = bucket.definitionId == null ? '' : `${bucket.definitionId}:`; return `${prefix}${bucket.bucket}/${bucket.table}/${bucket.id}`; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts index 5d05c63f8..51610a54b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts @@ -1,14 +1,8 @@ -import { mongo } from '@powersync/lib-service-mongodb'; -import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; -import { ReplicationAssertionError } from '@powersync/lib-services-framework'; -import { storage } from '@powersync/service-core'; - import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; import { CurrentDataStore } from './CurrentDataStore.js'; import { CurrentDataStoreV1 } from './CurrentDataStoreV1.js'; import { PersistedBatch } from './PersistedBatch.js'; import { PersistedBatchV1 } from './PersistedBatchV1.js'; -import { CommonCurrentBucket, CommonCurrentLookup, isCurrentBucketV3, isRecordedLookupV3 } from './models.js'; export class MongoBucketBatchV1 extends MongoBucketBatch { private readonly store: CurrentDataStore; @@ -24,18 +18,6 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { }); } - protected mapEvaluatedBuckets(evaluated: EvaluatedRow[]): CommonCurrentBucket[] { - return evaluated.map((entry) => ({ - bucket: entry.bucket, - table: entry.table, - id: entry.id - })); - } - - protected mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CommonCurrentLookup[] { - return paramEvaluated.map((entry) => storage.serializeLookup(entry.lookup)); - } - protected get currentDataStore(): CurrentDataStore { return this.store; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts index 5c4ad6198..d4546d801 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts @@ -1,20 +1,15 @@ -import * as bson from 'bson'; -import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; -import { storage } from '@powersync/service-core'; - import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; import { CurrentDataStore } from './CurrentDataStore.js'; import { CurrentDataStoreV3 } from './CurrentDataStoreV3.js'; import { PersistedBatch } from './PersistedBatch.js'; import { PersistedBatchV3 } from './PersistedBatchV3.js'; -import { CommonCurrentBucket, CommonCurrentLookup, CurrentBucketV3, RecordedLookupV3 } from './models.js'; export class MongoBucketBatchV3 extends MongoBucketBatch { private readonly store: CurrentDataStore; constructor(options: MongoBucketBatchOptions) { super(options); - this.store = new CurrentDataStoreV3(this.db, this.group_id); + this.store = new CurrentDataStoreV3(this.db, this.group_id, this.mapping); } protected createPersistedBatch(writtenSize: number): PersistedBatch { @@ -23,25 +18,6 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { }); } - protected mapEvaluatedBuckets(evaluated: EvaluatedRow[]): CommonCurrentBucket[] { - return evaluated.map((entry) => { - const def = this.mapping.bucketSourceId(entry.source); - return { - def, - bucket: entry.bucket, - table: entry.table, - id: entry.id - } satisfies CurrentBucketV3; - }); - } - - protected mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CommonCurrentLookup[] { - return paramEvaluated.map((entry) => { - const def = this.mapping.parameterLookupId(entry.lookup.source); - return { i: def, l: storage.serializeLookup(entry.lookup) } satisfies RecordedLookupV3; - }); - } - protected get currentDataStore(): CurrentDataStore { return this.store; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index fde1815b4..e9bb61544 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -7,16 +7,10 @@ import { InternalOpId, storage } from '@powersync/service-core'; import { MongoIdSequence } from './MongoIdSequence.js'; import { VersionedPowerSyncMongo } from './db.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { - BucketStateDocument, - CommonCurrentBucket, - CommonCurrentLookup, - CurrentDataDocumentId, - TaggedBucketParameterDocument, - TaggedBucketDataDocument -} from './models.js'; +import { BucketStateDocument, TaggedBucketParameterDocument, TaggedBucketDataDocument } from './models.js'; import { BucketDefinitionId } from './BucketDefinitionMapping.js'; import { mongoTableId } from '../../utils/util.js'; +import { CurrentDataBucketState, CurrentDataLookupState } from './CurrentDataStore.js'; /** * Maximum size of operations we write in a single transaction. @@ -44,7 +38,7 @@ export interface SaveBucketDataOptions { sourceKey: storage.ReplicaId; table: storage.SourceTable; evaluated: EvaluatedRow[]; - before_buckets: CommonCurrentBucket[]; + before_buckets: CurrentDataBucketState[]; } export interface SaveParameterDataOptions { @@ -52,15 +46,15 @@ export interface SaveParameterDataOptions { sourceKey: storage.ReplicaId; sourceTable: storage.SourceTable; evaluated: EvaluatedParameters[]; - existing_lookups: CommonCurrentLookup[]; + existing_lookups: CurrentDataLookupState[]; } export interface UpsertCurrentDataOptions { sourceTableId: bson.ObjectId; - id: CurrentDataDocumentId; + replicaId: storage.ReplicaId; data: bson.Binary | null; - buckets: CommonCurrentBucket[]; - lookups: CommonCurrentLookup[]; + buckets: CurrentDataBucketState[]; + lookups: CurrentDataLookupState[]; } export interface PersistedBatchOptions { @@ -104,11 +98,11 @@ export abstract class PersistedBatch { abstract saveParameterData(data: SaveParameterDataOptions): void; - abstract hardDeleteCurrentData(sourceTableId: bson.ObjectId, id: CurrentDataDocumentId): void; + abstract hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): void; abstract softDeleteCurrentData( sourceTableId: bson.ObjectId, - id: CurrentDataDocumentId, + replicaId: storage.ReplicaId, checkpointGreaterThan: bigint ): void; diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index 131cc1b24..d40c8f440 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -13,15 +13,12 @@ import { } from './PersistedBatch.js'; import { BucketParameterDocument, - CurrentBucket, CurrentDataDocument, LEGACY_BUCKET_DATA_DEFINITION_ID, LEGACY_BUCKET_PARAMETER_INDEX_ID, SourceKey, taggedBucketParameterDocumentToV1, - taggedBucketDataDocumentToV1, - isCurrentBucketV3, - isRecordedLookupV3 + taggedBucketDataDocumentToV1 } from './models.js'; import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; @@ -29,9 +26,9 @@ export class PersistedBatchV1 extends PersistedBatch { currentData: mongo.AnyBulkWriteOperation[] = []; saveBucketData(options: SaveBucketDataOptions) { - const remaining_buckets = new Map(); + const remaining_buckets = new Map(); for (let bucket of options.before_buckets) { - if (isCurrentBucketV3(bucket)) { + if (bucket.definitionId != null) { throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); } remaining_buckets.set(currentBucketKey(bucket), bucket); @@ -41,6 +38,7 @@ export class PersistedBatchV1 extends PersistedBatch { for (const evaluated of options.evaluated) { const key = currentBucketKey({ + definitionId: null, bucket: evaluated.bucket, table: evaluated.table, id: evaluated.id @@ -98,10 +96,10 @@ export class PersistedBatchV1 extends PersistedBatch { const remaining_lookups = new Map(); for (let lookup of data.existing_lookups) { - if (isRecordedLookupV3(lookup)) { + if (lookup.indexId != null) { throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); } - remaining_lookups.set(lookup.toString('base64'), lookup); + remaining_lookups.set(lookup.lookup.toString('base64'), lookup.lookup); } for (let result of evaluated) { @@ -150,36 +148,40 @@ export class PersistedBatchV1 extends PersistedBatch { } } - hardDeleteCurrentData(_sourceTableId: bson.ObjectId, id: SourceKey) { + hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) { this.currentData.push({ deleteOne: { - filter: { _id: id } + filter: { _id: this.currentDataId(sourceTableId, replicaId) } } }); this.currentSize += 50; } - softDeleteCurrentData(sourceTableId: bson.ObjectId, id: SourceKey, _checkpointGreaterThan: bigint) { - this.hardDeleteCurrentData(sourceTableId, id); + softDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId, _checkpointGreaterThan: bigint) { + this.hardDeleteCurrentData(sourceTableId, replicaId); } upsertCurrentData(values: UpsertCurrentDataOptions) { const buckets = values.buckets.map((bucket) => { - if (isCurrentBucketV3(bucket)) { + if (bucket.definitionId != null) { throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); } - return bucket; + return { + bucket: bucket.bucket, + table: bucket.table, + id: bucket.id + }; }); const lookups = values.lookups.map((lookup) => { - if (isRecordedLookupV3(lookup)) { + if (lookup.indexId != null) { throw new ReplicationAssertionError('Unexpected v3 lookup when incrementalReprocessing is disabled'); } - return lookup; + return lookup.lookup; }); this.currentData.push({ updateOne: { - filter: { _id: values.id }, + filter: { _id: this.currentDataId(values.sourceTableId, values.replicaId) }, update: { $set: { data: values.data ?? EMPTY_DATA, @@ -260,6 +262,14 @@ export class PersistedBatchV1 extends PersistedBatch { return this.getSourceTableId(operation)?.toHexString(); } + private currentDataId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey { + return { + g: this.group_id, + t: sourceTableId, + k: replicaId + }; + } + private getSourceTableId(operation: mongo.AnyBulkWriteOperation): bson.ObjectId | undefined { if ('updateOne' in operation) { return operation.updateOne.filter._id?.t; diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index d218e7542..79c34c55e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -14,12 +14,7 @@ import { } from './PersistedBatch.js'; import { BucketParameterDocumentV3, - CurrentBucketV3, - CurrentDataDocumentId, CurrentDataDocumentV3, - isCurrentBucketV3, - isRecordedLookupV3, - RecordedLookupV3, SourceTableKey, taggedBucketParameterDocumentToV3, taggedBucketDataDocumentToV3 @@ -29,9 +24,9 @@ export class PersistedBatchV3 extends PersistedBatch { currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation }[] = []; saveBucketData(options: SaveBucketDataOptions) { - const remaining_buckets = new Map(); + const remaining_buckets = new Map(); for (let bucket of options.before_buckets) { - if (!isCurrentBucketV3(bucket)) { + if (bucket.definitionId == null) { throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); } remaining_buckets.set(currentBucketKey(bucket), bucket); @@ -42,7 +37,7 @@ export class PersistedBatchV3 extends PersistedBatch { for (const evaluated of options.evaluated) { const sourceDefinitionId = this.mapping.bucketSourceId(evaluated.source); const key = currentBucketKey({ - def: sourceDefinitionId, + definitionId: sourceDefinitionId, bucket: evaluated.bucket, table: evaluated.table, id: evaluated.id @@ -79,10 +74,14 @@ export class PersistedBatchV3 extends PersistedBatch { for (let bucket of remaining_buckets.values()) { const op_id = options.op_seq.next(); this.debugLastOpId = op_id; + const definitionId = bucket.definitionId; + if (definitionId == null) { + throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); + } this.addBucketDataRemove({ op_id, - definitionId: bucket.def, + definitionId, bucket: bucket.bucket, sourceTableId: options.table.id, sourceKey: options.sourceKey, @@ -97,13 +96,13 @@ export class PersistedBatchV3 extends PersistedBatch { saveParameterData(data: SaveParameterDataOptions) { const { sourceTable, sourceKey, evaluated } = data; - const remaining_lookups = new Map(); + const remaining_lookups = new Map(); for (let lookup of data.existing_lookups) { - if (!isRecordedLookupV3(lookup)) { + if (lookup.indexId == null) { throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); } - remaining_lookups.set(`${lookup.i}.${lookup.l.toString('base64')}`, lookup); + remaining_lookups.set(`${lookup.indexId}.${lookup.lookup.toString('base64')}`, lookup); } for (let result of evaluated) { @@ -133,42 +132,46 @@ export class PersistedBatchV3 extends PersistedBatch { for (let lookup of remaining_lookups.values()) { const op_id = data.op_seq.next(); this.debugLastOpId = op_id; + const indexId = lookup.indexId; + if (indexId == null) { + throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); + } const values: BucketParameterDocumentV3 = { _id: op_id, key: { t: mongoTableId(sourceTable.id), k: sourceKey } satisfies SourceTableKey, - lookup: lookup.l, + lookup: lookup.lookup, bucket_parameters: [] }; this.bucketParameters.push({ ...values, - index: lookup.i + index: indexId }); this.currentSize += 200; } } - hardDeleteCurrentData(sourceTableId: bson.ObjectId, id: CurrentDataDocumentId) { + hardDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId) { this.currentData.push({ sourceTableId, operation: { deleteOne: { - filter: { _id: id } + filter: { _id: replicaId } } } }); this.currentSize += 50; } - softDeleteCurrentData(sourceTableId: bson.ObjectId, id: CurrentDataDocumentId, checkpointGreaterThan: bigint) { + softDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId, checkpointGreaterThan: bigint) { this.currentData.push({ sourceTableId, operation: { updateOne: { - filter: { _id: id }, + filter: { _id: replicaId }, update: { $set: { data: null, @@ -186,23 +189,31 @@ export class PersistedBatchV3 extends PersistedBatch { upsertCurrentData(values: UpsertCurrentDataOptions) { const buckets = values.buckets.map((bucket) => { - if (!isCurrentBucketV3(bucket)) { + if (bucket.definitionId == null) { throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); } - return bucket; + return { + def: bucket.definitionId, + bucket: bucket.bucket, + table: bucket.table, + id: bucket.id + }; }); const lookups = values.lookups.map((lookup) => { - if (!isRecordedLookupV3(lookup)) { + if (lookup.indexId == null) { throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled'); } - return lookup; + return { + i: lookup.indexId, + l: lookup.lookup + }; }); this.currentData.push({ sourceTableId: values.sourceTableId, operation: { updateOne: { - filter: { _id: values.id }, + filter: { _id: values.replicaId }, update: { $set: { data: values.data, From 15bd8836118c57133900bc3afcd04b585eb56cc6 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 12:14:48 +0200 Subject: [PATCH 28/93] Rename CurrentData -> SourceRecord. --- .../implementation/MongoBucketBatch.ts | 44 +++++++++---------- .../implementation/MongoBucketBatchShared.ts | 4 +- .../implementation/MongoBucketBatchV1.ts | 10 ++--- .../implementation/MongoBucketBatchV3.ts | 10 ++--- .../storage/implementation/PersistedBatch.ts | 10 ++--- ...rrentDataStore.ts => SourceRecordStore.ts} | 26 +++++------ ...tDataStoreV1.ts => SourceRecordStoreV1.ts} | 30 ++++++------- ...tDataStoreV3.ts => SourceRecordStoreV3.ts} | 22 +++++----- 8 files changed, 78 insertions(+), 78 deletions(-) rename modules/module-mongodb-storage/src/storage/implementation/{CurrentDataStore.ts => SourceRecordStore.ts} (58%) rename modules/module-mongodb-storage/src/storage/implementation/{CurrentDataStoreV1.ts => SourceRecordStoreV1.ts} (84%) rename modules/module-mongodb-storage/src/storage/implementation/{CurrentDataStoreV3.ts => SourceRecordStoreV3.ts} (86%) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 05849ae9a..e213da250 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -27,7 +27,7 @@ import * as timers from 'node:timers/promises'; import { mongoTableId } from '../../utils/util.js'; import { VersionedPowerSyncMongo } from './db.js'; import { SyncRuleDocument } from './models.js'; -import { CurrentDataStore, LoadedCurrentData } from './CurrentDataStore.js'; +import { LoadedSourceRecord, SourceRecordStore } from './SourceRecordStore.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { OperationBatch, RecordOperation } from './OperationBatch.js'; @@ -148,7 +148,7 @@ export abstract class MongoBucketBatch protected abstract createPersistedBatch(writtenSize: number): PersistedBatch; - protected abstract get currentDataStore(): CurrentDataStore; + protected abstract get sourceRecordStore(): SourceRecordStore; async flush(options?: storage.BatchBucketFlushOptions): Promise { let result: storage.FlushedResult | null = null; @@ -220,7 +220,7 @@ export abstract class MongoBucketBatch replicaId: r.beforeId })); - sizes = await this.currentDataStore.loadSizes(session, sizeLookups); + sizes = await this.sourceRecordStore.loadSizes(session, sizeLookups); } // If set, we need to start a new transaction with this batch. @@ -242,7 +242,7 @@ export abstract class MongoBucketBatch sourceTableId: mongoTableId(r.record.sourceTable.id), replicaId: r.beforeId })); - let current_data_lookup = await this.currentDataStore.loadDocuments(session, lookups, this.skipExistingRows); + let sourceRecordLookup = await this.sourceRecordStore.loadDocuments(session, lookups, this.skipExistingRows); let persistedBatch: PersistedBatch | null = this.createPersistedBatch(transactionSize); @@ -251,15 +251,15 @@ export abstract class MongoBucketBatch resumeBatch.push(op); continue; } - const currentData = current_data_lookup.get(op.internalBeforeKey) ?? null; - if (currentData != null) { + const sourceRecord = sourceRecordLookup.get(op.internalBeforeKey) ?? null; + if (sourceRecord != null) { // If it will be used again later, it will be set again using nextData below - current_data_lookup.delete(op.internalBeforeKey); + sourceRecordLookup.delete(op.internalBeforeKey); } - const nextData = this.saveOperation(persistedBatch!, op, currentData, op_seq); + const nextData = this.saveOperation(persistedBatch!, op, sourceRecord, op_seq); if (nextData != null) { // Update our current_data and size cache - current_data_lookup.set(op.internalAfterKey!, nextData); + sourceRecordLookup.set(op.internalAfterKey!, nextData); sizes?.set(op.internalAfterKey!, nextData.data?.length() ?? 0); } @@ -293,7 +293,7 @@ export abstract class MongoBucketBatch private saveOperation( batch: PersistedBatch, operation: RecordOperation, - current_data: LoadedCurrentData | null, + sourceRecord: LoadedSourceRecord | null, opSeq: MongoIdSequence ) { const record = operation.record; @@ -302,16 +302,16 @@ export abstract class MongoBucketBatch let after = record.after; const sourceTable = record.sourceTable; - let existing_buckets: LoadedCurrentData['buckets'] = []; - let new_buckets: LoadedCurrentData['buckets'] = []; - let existing_lookups: LoadedCurrentData['lookups'] = []; - let new_lookups: LoadedCurrentData['lookups'] = []; + let existing_buckets: LoadedSourceRecord['buckets'] = []; + let new_buckets: LoadedSourceRecord['buckets'] = []; + let existing_lookups: LoadedSourceRecord['lookups'] = []; + let new_lookups: LoadedSourceRecord['lookups'] = []; const sourceTableId = mongoTableId(record.sourceTable.id); if (this.skipExistingRows) { if (record.tag == SaveOperationTag.INSERT) { - if (current_data != null) { + if (sourceRecord != null) { // Initial replication, and we already have the record. // This may be a different version of the record, but streaming replication // will take care of that. @@ -324,7 +324,7 @@ export abstract class MongoBucketBatch } if (record.tag == SaveOperationTag.UPDATE) { - const result = current_data; + const result = sourceRecord; if (result == null) { // Not an error if we re-apply a transaction existing_buckets = []; @@ -351,7 +351,7 @@ export abstract class MongoBucketBatch } } } else if (record.tag == SaveOperationTag.DELETE) { - const result = current_data; + const result = sourceRecord; if (result == null) { // Not an error if we re-apply a transaction existing_buckets = []; @@ -468,7 +468,7 @@ export abstract class MongoBucketBatch table: sourceTable, before_buckets: existing_buckets }); - new_buckets = this.currentDataStore.mapEvaluatedBuckets(evaluated); + new_buckets = this.sourceRecordStore.mapEvaluatedBuckets(evaluated); } if (sourceTable.syncParameters) { @@ -501,11 +501,11 @@ export abstract class MongoBucketBatch evaluated: paramEvaluated, existing_lookups }); - new_lookups = this.currentDataStore.mapParameterLookups(paramEvaluated); + new_lookups = this.sourceRecordStore.mapParameterLookups(paramEvaluated); } } - let result: LoadedCurrentData | null = null; + let result: LoadedSourceRecord | null = null; // 5. TOAST: Update current data and bucket list. if (afterId) { @@ -814,7 +814,7 @@ export abstract class MongoBucketBatch this.persisted_op = null; this.last_checkpoint_lsn = lsn; if (newLastCheckpoint != null) { - await this.currentDataStore.cleanup(newLastCheckpoint, this.logger); + await this.sourceRecordStore.cleanup(newLastCheckpoint, this.logger); } } return { checkpointBlocked, checkpointCreated }; @@ -976,7 +976,7 @@ export abstract class MongoBucketBatch while (lastBatchCount == BATCH_LIMIT) { await this.withReplicationTransaction(`Truncate ${sourceTable.qualifiedName}`, async (session, opSeq) => { const sourceTableId = mongoTableId(sourceTable.id); - const batch = await this.currentDataStore.loadTruncateBatch(session, sourceTableId, BATCH_LIMIT); + const batch = await this.sourceRecordStore.loadTruncateBatch(session, sourceTableId, BATCH_LIMIT); const persistedBatch = this.createPersistedBatch(0); for (let value of batch) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts index ac15cf459..64c0e8427 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts @@ -1,11 +1,11 @@ import * as bson from 'bson'; -import { CurrentDataBucketState } from './CurrentDataStore.js'; +import { SourceRecordBucketState } from './SourceRecordStore.js'; export const MAX_ROW_SIZE = 15 * 1024 * 1024; export const EMPTY_DATA = new bson.Binary(bson.serialize({})); -export function currentBucketKey(bucket: CurrentDataBucketState) { +export function currentBucketKey(bucket: SourceRecordBucketState) { const prefix = bucket.definitionId == null ? '' : `${bucket.definitionId}:`; return `${prefix}${bucket.bucket}/${bucket.table}/${bucket.id}`; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts index 51610a54b..785b4300f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts @@ -1,15 +1,15 @@ import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; -import { CurrentDataStore } from './CurrentDataStore.js'; -import { CurrentDataStoreV1 } from './CurrentDataStoreV1.js'; +import { SourceRecordStore } from './SourceRecordStore.js'; +import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js'; import { PersistedBatch } from './PersistedBatch.js'; import { PersistedBatchV1 } from './PersistedBatchV1.js'; export class MongoBucketBatchV1 extends MongoBucketBatch { - private readonly store: CurrentDataStore; + private readonly store: SourceRecordStore; constructor(options: MongoBucketBatchOptions) { super(options); - this.store = new CurrentDataStoreV1(this.db, this.group_id); + this.store = new SourceRecordStoreV1(this.db, this.group_id); } protected createPersistedBatch(writtenSize: number): PersistedBatch { @@ -18,7 +18,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { }); } - protected get currentDataStore(): CurrentDataStore { + protected get sourceRecordStore(): SourceRecordStore { return this.store; } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts index d4546d801..b5eb2662f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts @@ -1,15 +1,15 @@ import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; -import { CurrentDataStore } from './CurrentDataStore.js'; -import { CurrentDataStoreV3 } from './CurrentDataStoreV3.js'; +import { SourceRecordStore } from './SourceRecordStore.js'; +import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js'; import { PersistedBatch } from './PersistedBatch.js'; import { PersistedBatchV3 } from './PersistedBatchV3.js'; export class MongoBucketBatchV3 extends MongoBucketBatch { - private readonly store: CurrentDataStore; + private readonly store: SourceRecordStore; constructor(options: MongoBucketBatchOptions) { super(options); - this.store = new CurrentDataStoreV3(this.db, this.group_id, this.mapping); + this.store = new SourceRecordStoreV3(this.db, this.group_id, this.mapping); } protected createPersistedBatch(writtenSize: number): PersistedBatch { @@ -18,7 +18,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { }); } - protected get currentDataStore(): CurrentDataStore { + protected get sourceRecordStore(): SourceRecordStore { return this.store; } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index e9bb61544..49f942317 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -10,7 +10,7 @@ import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { BucketStateDocument, TaggedBucketParameterDocument, TaggedBucketDataDocument } from './models.js'; import { BucketDefinitionId } from './BucketDefinitionMapping.js'; import { mongoTableId } from '../../utils/util.js'; -import { CurrentDataBucketState, CurrentDataLookupState } from './CurrentDataStore.js'; +import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js'; /** * Maximum size of operations we write in a single transaction. @@ -38,7 +38,7 @@ export interface SaveBucketDataOptions { sourceKey: storage.ReplicaId; table: storage.SourceTable; evaluated: EvaluatedRow[]; - before_buckets: CurrentDataBucketState[]; + before_buckets: SourceRecordBucketState[]; } export interface SaveParameterDataOptions { @@ -46,15 +46,15 @@ export interface SaveParameterDataOptions { sourceKey: storage.ReplicaId; sourceTable: storage.SourceTable; evaluated: EvaluatedParameters[]; - existing_lookups: CurrentDataLookupState[]; + existing_lookups: SourceRecordLookupState[]; } export interface UpsertCurrentDataOptions { sourceTableId: bson.ObjectId; replicaId: storage.ReplicaId; data: bson.Binary | null; - buckets: CurrentDataBucketState[]; - lookups: CurrentDataLookupState[]; + buckets: SourceRecordBucketState[]; + lookups: SourceRecordLookupState[]; } export interface PersistedBatchOptions { diff --git a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts similarity index 58% rename from modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts rename to modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts index b6eca0460..1b0386414 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStore.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts @@ -5,45 +5,45 @@ import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules import * as bson from 'bson'; import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; -export interface CurrentDataLookupEntry { +export interface SourceRecordLookupEntry { sourceTableId: bson.ObjectId; replicaId: storage.ReplicaId; } -export interface CurrentDataBucketState { +export interface SourceRecordBucketState { definitionId: BucketDefinitionId | null; bucket: string; table: string; id: string; } -export interface CurrentDataLookupState { +export interface SourceRecordLookupState { indexId: ParameterIndexId | null; lookup: bson.Binary; } -export interface LoadedCurrentData { +export interface LoadedSourceRecord { sourceTableId: bson.ObjectId; replicaId: storage.ReplicaId; data: bson.Binary | null; - buckets: CurrentDataBucketState[]; - lookups: CurrentDataLookupState[]; + buckets: SourceRecordBucketState[]; + lookups: SourceRecordLookupState[]; cacheKey: string; } -export interface CurrentDataStore { - mapEvaluatedBuckets(evaluated: EvaluatedRow[]): CurrentDataBucketState[]; - mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CurrentDataLookupState[]; - loadSizes(session: mongo.ClientSession, entries: CurrentDataLookupEntry[]): Promise>; +export interface SourceRecordStore { + mapEvaluatedBuckets(evaluated: EvaluatedRow[]): SourceRecordBucketState[]; + mapParameterLookups(paramEvaluated: EvaluatedParameters[]): SourceRecordLookupState[]; + loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise>; loadDocuments( session: mongo.ClientSession, - entries: CurrentDataLookupEntry[], + entries: SourceRecordLookupEntry[], idsOnly: boolean - ): Promise>; + ): Promise>; loadTruncateBatch( session: mongo.ClientSession, sourceTableId: bson.ObjectId, limit: number - ): Promise; + ): Promise; cleanup(lastCheckpoint: bigint, logger: Logger): Promise; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts similarity index 84% rename from modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts rename to modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts index 39913fd55..93d3d68e7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts @@ -6,21 +6,21 @@ import { idPrefixFilter } from '../../utils/util.js'; import { VersionedPowerSyncMongo } from './db.js'; import { cacheKey } from './OperationBatch.js'; import { - CurrentDataStore, - CurrentDataLookupEntry, - CurrentDataLookupState, - LoadedCurrentData -} from './CurrentDataStore.js'; + SourceRecordLookupEntry, + SourceRecordLookupState, + LoadedSourceRecord, + SourceRecordStore +} from './SourceRecordStore.js'; import { CurrentDataDocument, SourceKey } from './models.js'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; -export class CurrentDataStoreV1 implements CurrentDataStore { +export class SourceRecordStoreV1 implements SourceRecordStore { constructor( private readonly db: VersionedPowerSyncMongo, private readonly groupId: number ) {} - mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedCurrentData['buckets'] { + mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedSourceRecord['buckets'] { return evaluated.map((entry) => ({ definitionId: null, bucket: entry.bucket, @@ -29,7 +29,7 @@ export class CurrentDataStoreV1 implements CurrentDataStore { })); } - mapParameterLookups(paramEvaluated: EvaluatedParameters[]): CurrentDataLookupState[] { + mapParameterLookups(paramEvaluated: EvaluatedParameters[]): SourceRecordLookupState[] { return paramEvaluated.map((entry) => ({ indexId: null, lookup: storage.serializeLookup(entry.lookup) @@ -50,7 +50,7 @@ export class CurrentDataStoreV1 implements CurrentDataStore { data: bson.Binary | null, buckets: CurrentDataDocument['buckets'], lookups: CurrentDataDocument['lookups'] - ): LoadedCurrentData { + ): LoadedSourceRecord { return { sourceTableId, replicaId: id.k, @@ -69,7 +69,7 @@ export class CurrentDataStoreV1 implements CurrentDataStore { }; } - async loadSizes(session: mongo.ClientSession, entries: CurrentDataLookupEntry[]): Promise> { + async loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise> { const sizes = new Map(); for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { const sizeCursor: mongo.AggregationCursor = this.db @@ -101,10 +101,10 @@ export class CurrentDataStoreV1 implements CurrentDataStore { async loadDocuments( session: mongo.ClientSession, - entries: CurrentDataLookupEntry[], + entries: SourceRecordLookupEntry[], idsOnly: boolean - ): Promise> { - const documents = new Map(); + ): Promise> { + const documents = new Map(); const projection = idsOnly ? { _id: 1 } : undefined; for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { const cursor = this.db.v1_current_data(this.groupId, sourceTableId).find( @@ -133,7 +133,7 @@ export class CurrentDataStoreV1 implements CurrentDataStore { session: mongo.ClientSession, sourceTableId: bson.ObjectId, limit: number - ): Promise { + ): Promise { const cursor = this.db.v1_current_data(this.groupId, sourceTableId).find( { _id: idPrefixFilter({ g: this.groupId, t: sourceTableId }, ['k']), @@ -156,7 +156,7 @@ export class CurrentDataStoreV1 implements CurrentDataStore { async cleanup(_lastCheckpoint: bigint, _logger: Logger): Promise {} - private groupEntries(entries: CurrentDataLookupEntry[]): Map { + private groupEntries(entries: SourceRecordLookupEntry[]): Map { const grouped = new Map(); for (const entry of entries) { const existing = grouped.get(entry.sourceTableId) ?? []; diff --git a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts similarity index 86% rename from modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts rename to modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts index 7098e1b93..113f8937b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/CurrentDataStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts @@ -5,18 +5,18 @@ import * as bson from 'bson'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import { VersionedPowerSyncMongo } from './db.js'; import { cacheKey } from './OperationBatch.js'; -import { CurrentDataStore, CurrentDataLookupEntry, LoadedCurrentData } from './CurrentDataStore.js'; +import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from './SourceRecordStore.js'; import { CurrentDataDocumentV3 } from './models.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -export class CurrentDataStoreV3 implements CurrentDataStore { +export class SourceRecordStoreV3 implements SourceRecordStore { constructor( private readonly db: VersionedPowerSyncMongo, private readonly groupId: number, private readonly mapping: BucketDefinitionMapping ) {} - mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedCurrentData['buckets'] { + mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedSourceRecord['buckets'] { return evaluated.map((entry) => ({ definitionId: this.mapping.bucketSourceId(entry.source), bucket: entry.bucket, @@ -25,7 +25,7 @@ export class CurrentDataStoreV3 implements CurrentDataStore { })); } - mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedCurrentData['lookups'] { + mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedSourceRecord['lookups'] { return paramEvaluated.map((entry) => ({ indexId: this.mapping.parameterLookupId(entry.lookup.source), lookup: storage.serializeLookup(entry.lookup) @@ -38,7 +38,7 @@ export class CurrentDataStoreV3 implements CurrentDataStore { data: bson.Binary | null, buckets: CurrentDataDocumentV3['buckets'], lookups: CurrentDataDocumentV3['lookups'] - ): LoadedCurrentData { + ): LoadedSourceRecord { return { sourceTableId, replicaId: id, @@ -57,7 +57,7 @@ export class CurrentDataStoreV3 implements CurrentDataStore { }; } - async loadSizes(session: mongo.ClientSession, entries: CurrentDataLookupEntry[]): Promise> { + async loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise> { const sizes = new Map(); for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { const filter = { @@ -88,10 +88,10 @@ export class CurrentDataStoreV3 implements CurrentDataStore { async loadDocuments( session: mongo.ClientSession, - entries: CurrentDataLookupEntry[], + entries: SourceRecordLookupEntry[], idsOnly: boolean - ): Promise> { - const documents = new Map(); + ): Promise> { + const documents = new Map(); const projection = idsOnly ? { _id: 1 } : undefined; for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { const filter = { @@ -116,7 +116,7 @@ export class CurrentDataStoreV3 implements CurrentDataStore { session: mongo.ClientSession, sourceTableId: bson.ObjectId, limit: number - ): Promise { + ): Promise { const cursor = this.db.v3_current_data(this.groupId, sourceTableId).find( { pending_delete: { $exists: false } @@ -149,7 +149,7 @@ export class CurrentDataStoreV3 implements CurrentDataStore { } } - private groupEntries(entries: CurrentDataLookupEntry[]): Map { + private groupEntries(entries: SourceRecordLookupEntry[]): Map { const grouped = new Map(); for (const entry of entries) { const existing = grouped.get(entry.sourceTableId) ?? []; From 600a10d7a975cd2b10afa1bee9549840c880dff3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 12:40:35 +0200 Subject: [PATCH 29/93] Split out source_tables collections. --- .../implementation/MongoBucketBatch.ts | 6 +- .../implementation/MongoSyncBucketStorage.ts | 18 ++-- .../src/storage/implementation/db.ts | 85 +++++++++++++------ 3 files changed, 71 insertions(+), 38 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index e213da250..c6a0e2dff 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -941,7 +941,7 @@ export abstract class MongoBucketBatch await this.withTransaction(async () => { for (let table of sourceTables) { - await this.db.source_tables.deleteOne({ _id: mongoTableId(table.id) }); + await this.db.source_tables(this.group_id).deleteOne({ _id: mongoTableId(table.id) }); } }); return result; @@ -1021,7 +1021,7 @@ export abstract class MongoBucketBatch copy.snapshotStatus = snapshotStatus; await this.withTransaction(async () => { - await this.db.source_tables.updateOne( + await this.db.source_tables(this.group_id).updateOne( { _id: mongoTableId(table.id) }, { $set: { @@ -1076,7 +1076,7 @@ export abstract class MongoBucketBatch const ids = tables.map((table) => mongoTableId(table.id)); await this.withTransaction(async () => { - await this.db.source_tables.updateMany( + await this.db.source_tables(this.group_id).updateMany( { _id: { $in: ids } }, { $set: { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index a995eee07..facc3285e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -261,10 +261,11 @@ export class MongoSyncBucketStorage type: column.type, type_oid: column.typeId })); + await this.db.initializeSourceTablesCollection(group_id); const mapping = this.sync_rules.mapping; let result: storage.ResolveTableResult | null = null; await this.db.client.withSession(async (session) => { - const col = this.db.source_tables; + const col = this.db.source_tables(group_id); let filter: Partial = { group_id: group_id, connection_id: connection_id, @@ -992,12 +993,15 @@ export class MongoSyncBucketStorage { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); - await this.db.source_tables.deleteMany( - { - group_id: this.group_id - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); + await this.db + .source_tables(this.group_id) + .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); this.#storageInitialized = false; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index e0292b4a9..11fc016a0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -41,8 +41,6 @@ export class PowerSyncMongo { readonly bucket_parameters: mongo.Collection; readonly op_id_sequence: mongo.Collection; readonly sync_rules: mongo.Collection; - readonly source_tables: mongo.Collection; - readonly v3_source_tables: mongo.Collection; readonly custom_write_checkpoints: mongo.Collection; readonly write_checkpoints: mongo.Collection; readonly instance: mongo.Collection; @@ -68,8 +66,6 @@ export class PowerSyncMongo { this.bucket_parameters = db.collection('bucket_parameters'); this.op_id_sequence = db.collection('op_id_sequence'); this.sync_rules = db.collection('sync_rules'); - this.source_tables = db.collection('source_tables'); - this.v3_source_tables = db.collection('v3_source_tables'); this.custom_write_checkpoints = db.collection('custom_write_checkpoints'); this.write_checkpoints = db.collection('write_checkpoints'); this.instance = db.collection('instance'); @@ -132,6 +128,10 @@ export class PowerSyncMongo { return `source_records_${replicationStreamId}_${sourceTableId.toHexString()}`; } + sourceTableCollectionName(replicationStreamId: number) { + return `source_table_${replicationStreamId}`; + } + sourceRecords( replicationStreamId: number, sourceTableId: mongo.ObjectId @@ -150,6 +150,25 @@ export class PowerSyncMongo { .map((collection) => this.db.collection(collection.name)); } + sourceTables(replicationStreamId: number): mongo.Collection { + return this.db.collection(this.sourceTableCollectionName(replicationStreamId)); + } + + async listSourceTableCollections( + replicationStreamId?: number + ): Promise[]> { + const filter = + replicationStreamId == null + ? { name: new RegExp('^source_table_') } + : { name: this.sourceTableCollectionName(replicationStreamId) }; + const prefix = replicationStreamId == null ? 'source_table_' : this.sourceTableCollectionName(replicationStreamId); + const collections = await this.db.listCollections(filter, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); + } + /** * Clear all collections. */ @@ -169,8 +188,17 @@ export class PowerSyncMongo { } await this.op_id_sequence.deleteMany({}); await this.sync_rules.deleteMany({}); - await this.source_tables.deleteMany({}); - await this.v3_source_tables.deleteMany({}); + for (const collection of await this.listSourceTableCollections()) { + await collection.drop(); + } + for (const legacyName of ['source_tables', 'v3_source_tables']) { + await this.db.dropCollection(legacyName).catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } await this.write_checkpoints.deleteMany({}); await this.instance.deleteOne({}); await this.locks.deleteMany({}); @@ -268,20 +296,21 @@ export class PowerSyncMongo { } async initializeStorageVersion(storageConfig: StorageConfig) { - if (storageConfig.incrementalReprocessing) { - await this.v3_source_tables.createIndex( - { - group_id: 1, - connection_id: 1, - schema_name: 1, - table_name: 1, - relation_id: 1 - }, - { - name: 'source_lookup' - } - ); - } + // Per-stream collections are initialized lazily when first accessed. + } + + async initializeSourceTablesCollection(replicationStreamId: number) { + await this.sourceTables(replicationStreamId).createIndex( + { + connection_id: 1, + schema_name: 1, + table_name: 1, + relation_id: 1 + }, + { + name: 'source_lookup' + } + ); } async initializeSourceRecordsCollection( @@ -360,6 +389,14 @@ export class VersionedPowerSyncMongo { return this.#upstream.initializeSourceRecordsCollection(this.storageConfig, replicationStreamId, sourceTableId); } + source_tables(replicationStreamId: number): mongo.Collection { + return this.#upstream.sourceTables(replicationStreamId); + } + + initializeSourceTablesCollection(replicationStreamId: number) { + return this.#upstream.initializeSourceTablesCollection(replicationStreamId); + } + get bucket_data() { return this.#upstream.bucket_data; } @@ -426,14 +463,6 @@ export class VersionedPowerSyncMongo { return this.#upstream.sync_rules; } - get source_tables() { - if (this.storageConfig.incrementalReprocessing) { - return this.#upstream.v3_source_tables as unknown as mongo.Collection; - } else { - return this.#upstream.source_tables as unknown as mongo.Collection; - } - } - get custom_write_checkpoints() { return this.#upstream.custom_write_checkpoints; } From bdb8061b75afd0a38967341820c1f987e970b96c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 12:49:05 +0200 Subject: [PATCH 30/93] Don't do initializeCurrentDataCollection for v1. --- .../storage/implementation/MongoBucketBatch.ts | 15 +++++++++++++++ .../storage/implementation/PersistedBatchV1.ts | 1 - .../src/storage/implementation/db.ts | 17 +++++++---------- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index c6a0e2dff..a71e49ede 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -1,3 +1,4 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { HydratedSyncRules, SqlEventDescriptor, SqliteRow, SqliteValue } from '@powersync/service-sync-rules'; import * as bson from 'bson'; @@ -944,6 +945,20 @@ export abstract class MongoBucketBatch await this.db.source_tables(this.group_id).deleteOne({ _id: mongoTableId(table.id) }); } }); + + if (this.db.storageConfig.incrementalReprocessing) { + for (let table of sourceTables) { + await this.db + .common_current_data(this.group_id, mongoTableId(table.id)) + .drop() + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } + } return result; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index d40c8f440..8f60145d6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -246,7 +246,6 @@ export class PersistedBatchV1 extends PersistedBatch { if (sourceTableId == null) { throw new ReplicationAssertionError('Missing source table id for current_data bulkWrite'); } - await this.db.initializeCurrentDataCollection(this.group_id, sourceTableId); await this.db.v1_current_data(this.group_id, sourceTableId).bulkWrite(operations, { session, ordered: true diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 11fc016a0..195a466fc 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -313,15 +313,7 @@ export class PowerSyncMongo { ); } - async initializeSourceRecordsCollection( - storageConfig: StorageConfig, - replicationStreamId: number, - sourceTableId: mongo.ObjectId - ) { - if (!storageConfig.incrementalReprocessing) { - return; - } - + async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { await this.sourceRecords(replicationStreamId, sourceTableId).createIndex( { pending_delete: 1 @@ -386,7 +378,12 @@ export class VersionedPowerSyncMongo { } initializeCurrentDataCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { - return this.#upstream.initializeSourceRecordsCollection(this.storageConfig, replicationStreamId, sourceTableId); + if (!this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'source_records collection initialization should not be used when incrementalReprocessing is disabled' + ); + } + return this.#upstream.initializeSourceRecordsCollection(replicationStreamId, sourceTableId); } source_tables(replicationStreamId: number): mongo.Collection { From f9a39c4550d8af6eebb5a848be92819c17f45c2c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 12:52:11 +0200 Subject: [PATCH 31/93] Initialize source records collection on resolveTable instead of flush. --- .../src/storage/implementation/MongoSyncBucketStorage.ts | 7 +++++++ .../src/storage/implementation/PersistedBatchV3.ts | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index facc3285e..a44b5519d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -264,6 +264,7 @@ export class MongoSyncBucketStorage await this.db.initializeSourceTablesCollection(group_id); const mapping = this.sync_rules.mapping; let result: storage.ResolveTableResult | null = null; + let initializeSourceRecordsFor: bson.ObjectId | null = null; await this.db.client.withSession(async (session) => { const col = this.db.source_tables(group_id); let filter: Partial = { @@ -315,6 +316,9 @@ export class MongoSyncBucketStorage doc = createDoc; await col.insertOne(doc, { session }); + if (this.db.storageConfig.incrementalReprocessing) { + initializeSourceRecordsFor = doc._id; + } } const sourceTable = new storage.SourceTable({ id: doc._id, @@ -374,6 +378,9 @@ export class MongoSyncBucketStorage dropTables: dropTables }; }); + if (initializeSourceRecordsFor != null) { + await this.db.initializeCurrentDataCollection(group_id, initializeSourceRecordsFor); + } return result!; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 79c34c55e..dc9c6c528 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -290,7 +290,6 @@ export class PersistedBatchV3 extends PersistedBatch { for (const operations of operationsBySourceTable.values()) { const sourceTableId = operations[0]!.sourceTableId; - await this.db.initializeCurrentDataCollection(this.group_id, sourceTableId); await this.db.v3_current_data(this.group_id, sourceTableId).bulkWrite( operations.map((entry) => entry.operation), { From f46a8a7fb99a10bddd9dcc77d5870282c15189a3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 13:03:07 +0200 Subject: [PATCH 32/93] Refactor more collection initialization. --- .../src/storage/MongoBucketStorage.ts | 1 - .../implementation/MongoSyncBucketStorage.ts | 5 +- .../src/storage/implementation/db.ts | 56 ++++++++----------- 3 files changed, 25 insertions(+), 37 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 9c50e8110..9169a31bb 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -156,7 +156,6 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION; const storageConfig = getMongoStorageConfig(storageVersion); - await this.db.initializeStorageVersion(storageConfig); let rules: MongoPersistedSyncRulesContent | undefined = undefined; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index a44b5519d..6c162e93e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -179,6 +179,8 @@ export class MongoSyncBucketStorage return; } + await this.db.initializeStreamStorage(this.group_id); + const mapping = this.sync_rules.mapping; for (let source of mapping.allBucketDefinitionIds()) { const collection = this.db.bucket_data_v3(this.group_id, source).collectionName; @@ -261,7 +263,6 @@ export class MongoSyncBucketStorage type: column.type, type_oid: column.typeId })); - await this.db.initializeSourceTablesCollection(group_id); const mapping = this.sync_rules.mapping; let result: storage.ResolveTableResult | null = null; let initializeSourceRecordsFor: bson.ObjectId | null = null; @@ -379,7 +380,7 @@ export class MongoSyncBucketStorage }; }); if (initializeSourceRecordsFor != null) { - await this.db.initializeCurrentDataCollection(group_id, initializeSourceRecordsFor); + await this.db.initializeSourceRecordsCollection(group_id, initializeSourceRecordsFor); } return result!; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 195a466fc..20751946a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -294,36 +294,6 @@ export class PowerSyncMongo { { name: 'dirty_count' } ); } - - async initializeStorageVersion(storageConfig: StorageConfig) { - // Per-stream collections are initialized lazily when first accessed. - } - - async initializeSourceTablesCollection(replicationStreamId: number) { - await this.sourceTables(replicationStreamId).createIndex( - { - connection_id: 1, - schema_name: 1, - table_name: 1, - relation_id: 1 - }, - { - name: 'source_lookup' - } - ); - } - - async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { - await this.sourceRecords(replicationStreamId, sourceTableId).createIndex( - { - pending_delete: 1 - }, - { - partialFilterExpression: { pending_delete: { $exists: true } }, - name: 'pending_delete' - } - ); - } } /** @@ -377,21 +347,39 @@ export class VersionedPowerSyncMongo { return this.#upstream.listSourceRecordCollections(replicationStreamId); } - initializeCurrentDataCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'source_records collection initialization should not be used when incrementalReprocessing is disabled' ); } - return this.#upstream.initializeSourceRecordsCollection(replicationStreamId, sourceTableId); + await this.#upstream.sourceRecords(replicationStreamId, sourceTableId).createIndex( + { + pending_delete: 1 + }, + { + partialFilterExpression: { pending_delete: { $exists: true } }, + name: 'pending_delete' + } + ); } source_tables(replicationStreamId: number): mongo.Collection { return this.#upstream.sourceTables(replicationStreamId); } - initializeSourceTablesCollection(replicationStreamId: number) { - return this.#upstream.initializeSourceTablesCollection(replicationStreamId); + async initializeStreamStorage(replicationStreamId: number) { + await this.source_tables(replicationStreamId).createIndex( + { + connection_id: 1, + schema_name: 1, + table_name: 1, + relation_id: 1 + }, + { + name: 'source_lookup' + } + ); } get bucket_data() { From e2ae05c18fa103c786e13bb2a8a93ebce6ea8b36 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 13:32:36 +0200 Subject: [PATCH 33/93] Restructure v3 parameter index lookup values. --- .../implementation/BucketDefinitionMapping.ts | 16 +++++------- .../implementation/MongoParameterLookupV3.ts | 12 +++++++++ .../implementation/MongoPersistedSyncRules.ts | 26 ++++++++++++++----- .../implementation/MongoSyncBucketStorage.ts | 19 +++++++------- .../implementation/PersistedBatchV3.ts | 3 ++- .../implementation/SourceRecordStoreV3.ts | 3 ++- .../test/src/storage_sync.test.ts | 23 +++++++++++++--- packages/service-core/src/storage/bson.ts | 5 ---- .../sync-rules/src/BucketParameterQuerier.ts | 11 ++++++++ 9 files changed, 81 insertions(+), 37 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoParameterLookupV3.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts index 9740eaaac..5b6a52db6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -1,5 +1,10 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { BucketDataSource, ParameterIndexLookupCreator, SyncConfigWithErrors } from '@powersync/service-sync-rules'; +import { + BucketDataSource, + ParameterIndexLookupCreator, + ParameterLookupScope, + SyncConfigWithErrors +} from '@powersync/service-sync-rules'; import { SyncRuleDocument } from './models.js'; export type BucketDefinitionId = string; @@ -59,15 +64,6 @@ export class BucketDefinitionMapping { return defId; } - parameterLookupScopeId(scope: Pick) { - const key = this.parameterLookupKey(scope.lookupName, scope.queryId); - const defId = this.parameterLookupMapping[key]; - if (defId == null) { - throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`); - } - return defId; - } - private parameterLookupKey(lookupName: string, queryId: string) { return `${lookupName}#${queryId}`; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterLookupV3.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterLookupV3.ts new file mode 100644 index 000000000..3742fbd5c --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterLookupV3.ts @@ -0,0 +1,12 @@ +import * as bson from 'bson'; +import { deserializeParameterLookup } from '@powersync/service-core'; +import { ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules'; +import { ParameterIndexId } from './BucketDefinitionMapping.js'; + +export function serializeParameterLookupV3(lookup: ScopedParameterLookup): bson.Binary { + return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) })); +} + +export function deserializeParameterLookupV3(lookup: bson.Binary, indexId: ParameterIndexId): SqliteJsonValue[] { + return [indexId, '', ...deserializeParameterLookup(lookup)]; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index c3b4870a0..ac496b55a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -12,6 +12,7 @@ import { import { storage } from '@powersync/service-core'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { StorageConfig } from './models.js'; +import { ServiceAssertionError } from '@powersync/lib-services-framework'; export class MongoPersistedSyncRules implements storage.PersistedSyncRules { public readonly hydrationState: HydrationState; @@ -23,10 +24,11 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules { private readonly mapping: BucketDefinitionMapping | null, private readonly storageConfig: StorageConfig ) { - if (this.mapping != null && this.storageConfig.incrementalReprocessing) { - // FIXME: Recheck bucket name generation again when we get to merging sync config versions. - // this.hydrationState = new MongoHydrationState(this.mapping); - this.hydrationState = versionedHydrationState(this.id); + if (this.storageConfig.incrementalReprocessing) { + if (this.mapping == null) { + throw new ServiceAssertionError(`mapping is required for v3 storage`); + } + this.hydrationState = new MongoHydrationState(this.mapping, this.id); } else if ( !this.sync_rules.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && !this.storageConfig.versionedBuckets @@ -43,12 +45,22 @@ export class MongoPersistedSyncRules implements storage.PersistedSyncRules { } class MongoHydrationState implements HydrationState { - constructor(private readonly mapping: BucketDefinitionMapping) {} + constructor( + private readonly mapping: BucketDefinitionMapping, + private readonly version: number + ) {} getBucketSourceScope(source: BucketDataSource): BucketDataScope { - const defId = this.mapping.bucketSourceId(source); + // Keep this aligned with versionedHydrationState() for now. + // + // Previous Mongo-specific behavior: + // const defId = this.mapping.bucketSourceId(source); + // return { + // bucketPrefix: defId, + // source + // }; return { - bucketPrefix: defId, + bucketPrefix: `${this.version}#${source.uniqueName}`, source }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 6c162e93e..977d3479c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -51,6 +51,7 @@ import { MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; +import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; export interface MongoSyncBucketStorageOptions { checksumOptions?: Omit; @@ -481,13 +482,9 @@ export class MongoSyncBucketStorage collection: mongo.Collection; pipeline: mongo.Document[]; } => { - const [lookupName, queryId] = lookup.values; - if (typeof lookupName != 'string' || typeof queryId != 'string') { - throw new ServiceAssertionError('Invalid scoped parameter lookup identifier'); - } - const indexId = this.sync_rules.mapping.parameterLookupScopeId({ lookupName, queryId }); + const indexId = lookup.indexId; const collection = this.db.bucket_parameters_v3(this.group_id, indexId); - const lookupFilter = storage.serializeLookup(lookup); + const lookupFilter = serializeParameterLookupV3(lookup); return { collection, pipeline: [ @@ -1354,14 +1351,16 @@ export class MongoSyncBucketStorage options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; - const parameterUpdates: { lookup: bson.Binary }[] = []; + const parameterUpdates: { lookup: bson.Binary; indexId: string }[] = []; + // FIXME: Optimize performance for many collections for (const collection of await this.db.listBucketParameterCollectionsV3(this.group_id)) { if (parameterUpdates.length > limit) { break; } const remaining = limit + 1 - parameterUpdates.length; + const indexId = collection.collectionName.slice(`parameter_index_${this.group_id}_`.length); const updates = await collection .find( { @@ -1377,7 +1376,7 @@ export class MongoSyncBucketStorage } ) .toArray(); - parameterUpdates.push(...updates); + parameterUpdates.push(...updates.map((update) => ({ ...update, indexId }))); } const invalidateParameterUpdates = parameterUpdates.length > limit; @@ -1386,7 +1385,9 @@ export class MongoSyncBucketStorage invalidateParameterBuckets: invalidateParameterUpdates, updatedParameterLookups: invalidateParameterUpdates ? new Set() - : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup)))) + : new Set( + parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookupV3(p.lookup, p.indexId))) + ) }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index dc9c6c528..3a18f2b0d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -19,6 +19,7 @@ import { taggedBucketParameterDocumentToV3, taggedBucketDataDocumentToV3 } from './models.js'; +import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; export class PersistedBatchV3 extends PersistedBatch { currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation }[] = []; @@ -107,7 +108,7 @@ export class PersistedBatchV3 extends PersistedBatch { for (let result of evaluated) { const sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source); - const binLookup = storage.serializeLookup(result.lookup); + const binLookup = serializeParameterLookupV3(result.lookup); remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`); const op_id = data.op_seq.next(); diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts index 113f8937b..e462db862 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts @@ -8,6 +8,7 @@ import { cacheKey } from './OperationBatch.js'; import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from './SourceRecordStore.js'; import { CurrentDataDocumentV3 } from './models.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; export class SourceRecordStoreV3 implements SourceRecordStore { constructor( @@ -28,7 +29,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore { mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedSourceRecord['lookups'] { return paramEvaluated.map((entry) => ({ indexId: this.mapping.parameterLookupId(entry.lookup.source), - lookup: storage.serializeLookup(entry.lookup) + lookup: serializeParameterLookupV3(entry.lookup) })); } diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index cdbced7d3..27008f85d 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -1,6 +1,7 @@ -import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { deserializeParameterLookup, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; +import { ScopedParameterLookup } from '@powersync/service-sync-rules'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; import { CurrentBucketV3, CurrentDataDocumentV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; @@ -158,8 +159,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor ` bucket_definitions: global: + parameters: + - SELECT owner_id FROM "%" WHERE id = token_parameters.id data: - - SELECT id, description FROM "%" + - SELECT id, description, owner_id FROM "%" `, { storageVersion } ) @@ -173,11 +176,18 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor tag: storage.SaveOperationTag.INSERT, after: { id: 'shape-check', - description: 'shape' + description: 'shape', + owner_id: 'user-1' }, afterReplicaId: test_utils.rid('shape-check') }); - await writer.flush(); + await writer.commit('1/1'); + + const checkpoint = await bucketStorage.getCheckpoint(); + const parameters = await checkpoint.getParameterSets([ + ScopedParameterLookup.direct({ lookupName: 'global', queryId: '1', source: null as any }, ['shape-check']) + ]); + expect(parameters).toEqual([{ owner_id: 'user-1' }]); const mongoFactory = factory as MongoBucketStorage; const currentDataCollections = await mongoFactory.db.listSourceRecordCollections(syncRules.id); @@ -195,6 +205,11 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const syncRule = await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id }); const ruleMapping: SyncRuleDocument['rule_mapping'] | undefined = syncRule?.rule_mapping; expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0); + + const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0]; + expect(parameterIndexId).toBeDefined(); + const parameterEntry = await mongoFactory.db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({}); + expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']); }); } diff --git a/packages/service-core/src/storage/bson.ts b/packages/service-core/src/storage/bson.ts index ad7ee3e16..831e1da30 100644 --- a/packages/service-core/src/storage/bson.ts +++ b/packages/service-core/src/storage/bson.ts @@ -40,11 +40,6 @@ export const deserializeParameterLookup = (lookup: bson.Binary) => { return parsed; }; -export const getLookupBucketDefinitionName = (lookup: bson.Binary) => { - const parsed = deserializeParameterLookup(lookup); - return parsed[0] as string; -}; - /** * True if this is a bson.UUID. * diff --git a/packages/sync-rules/src/BucketParameterQuerier.ts b/packages/sync-rules/src/BucketParameterQuerier.ts index de53c482c..e49a22967 100644 --- a/packages/sync-rules/src/BucketParameterQuerier.ts +++ b/packages/sync-rules/src/BucketParameterQuerier.ts @@ -120,6 +120,17 @@ export class ScopedParameterLookup { return (this.#cachedSerializedForm ??= JSONBig.stringify(this.values)); } + get indexId(): string { + const indexId = this.values[0]; + // TODO: Consider restructuring so that these values aren't present at all + if (this.values[1] != '') { + throw new Error('Unexpected queryId'); + } else if (typeof indexId != 'string') { + throw new Error('Unexpected indexId'); + } + return indexId; + } + static normalized(scope: ParameterLookupScope, lookup: UnscopedParameterLookup): ScopedParameterLookup { return new ScopedParameterLookup(scope.source, [scope.lookupName, scope.queryId, ...lookup.lookupValues]); } From 2c5a1ec6a4564518e5a2a9400b157873b8156a6c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 15:21:03 +0200 Subject: [PATCH 34/93] Update tests. --- .../register-data-storage-parameter-tests.ts | 195 +++++++++++++----- .../sync-rules/src/BucketParameterQuerier.ts | 14 ++ packages/sync-rules/src/HydrationState.ts | 6 +- 3 files changed, 157 insertions(+), 58 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index d9ddc2475..07208d495 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -3,7 +3,6 @@ import { RequestParameters, ScopedParameterLookup, SqliteJsonRow } from '@powers import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest } from '../test-utils/test-utils-index.js'; -import { parameterLookupScope } from './util.js'; /** * @example @@ -18,7 +17,6 @@ import { parameterLookupScope } from './util.js'; export function registerDataStorageParameterTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION; - const MYBUCKET_1 = parameterLookupScope('mybucket', '1'); test('save and load parameters', async () => { await using factory = await generateStorageFactory(); @@ -37,6 +35,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -69,12 +68,20 @@ bucket_definitions: await writer.commit('1/1'); const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); - expect(parameters).toEqual([ - { - group_id: 'group1a' + const parameters = new RequestParameters(new JwtPayload({ sub: 'user1' }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ group_id: 'group1a' }]); + return parameter_sets; } - ]); + }); + + expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group1a"]').bucket]); }); test('it should use the latest version', async () => { @@ -94,6 +101,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -121,20 +129,30 @@ bucket_definitions: await writer.commit('1/2'); const checkpoint2 = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint2.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); - expect(parameters).toEqual([ - { - group_id: 'group2' + const parameters = new RequestParameters(new JwtPayload({ sub: 'user1' }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + + const buckets1 = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]); + + const parameter_sets = await checkpoint1.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ group_id: 'group1' }]); + return parameter_sets; } - ]); + }); + expect(buckets1.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group1"]').bucket]); - // Use the checkpoint to get older data if relevant - const parameters2 = await checkpoint1.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); - expect(parameters2).toEqual([ - { - group_id: 'group1' + const buckets2 = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]); + + const parameter_sets = await checkpoint2.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ group_id: 'group2' }]); + return parameter_sets; } - ]); + }); + expect(buckets2.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group2"]').bucket]); }); test('it should use the latest version after updates', async () => { @@ -154,6 +172,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const table = await test_utils.resolveTestTable(writer, 'todos', ['id', 'list_id'], config); @@ -197,18 +216,28 @@ bucket_definitions: // There removal operation for the association of `list2`::`todo2` should not interfere with the new // association of `list1`::`todo2` const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(MYBUCKET_1, ['list1']), - ScopedParameterLookup.direct(MYBUCKET_1, ['list2']) - ]); + const parameters = new RequestParameters( + new JwtPayload({ sub: 'u1', parameters: { list_id: ['list1', 'list2'] } }), + {} + ); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - expect(parameters.sort((a, b) => (a.todo_id as string).localeCompare(b.todo_id as string))).toEqual([ - { - todo_id: 'todo1' - }, - { - todo_id: 'todo2' + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => JSON.stringify(l.indexKey)).sort()).toEqual(['["list1"]', '["list2"]']); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets.sort((a, b) => (a.todo_id as string).localeCompare(b.todo_id as string))).toEqual([ + { todo_id: 'todo1' }, + { todo_id: 'todo2' } + ]); + return parameter_sets; } + }); + + expect(buckets.map((b) => b.bucket).sort()).toEqual([ + bucketRequest(syncRules, 'mybucket["todo1"]').bucket, + bucketRequest(syncRules, 'mybucket["todo2"]').bucket ]); }); @@ -229,6 +258,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -248,20 +278,27 @@ bucket_definitions: await writer.commit('1/1'); - const TEST_PARAMS = { group_id: 'group1' }; - const checkpoint = await bucketStorage.getCheckpoint(); + const testQuery = async (jwtParameters: Record, expectedParameterSets: SqliteJsonRow[]) => { + const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: jwtParameters }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; - const parameters1 = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3.14]) + return await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual(expectedParameterSets); + return parameter_sets; + } + }); + }; + + expect(await testQuery({ n1: 314n, f2: 314, f3: 3.14 }, [{ group_id: 'group1' }])).toMatchObject([ + { bucket: bucketRequest(syncRules, 'mybucket["group1"]').bucket } ]); - expect(parameters1).toEqual([TEST_PARAMS]); - const parameters2 = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(MYBUCKET_1, [314, 314n, 3.14]) + expect(await testQuery({ n1: 314, f2: 314n, f3: 3.14 }, [{ group_id: 'group1' }])).toMatchObject([ + { bucket: bucketRequest(syncRules, 'mybucket["group1"]').bucket } ]); - expect(parameters2).toEqual([TEST_PARAMS]); - const parameters3 = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, [314n, 314, 3])]); - expect(parameters3).toEqual([]); + expect(await testQuery({ n1: 314n, f2: 314, f3: 3 }, [])).toEqual([]); }); test('save and load parameters with large numbers', async () => { @@ -285,6 +322,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -315,14 +353,23 @@ bucket_definitions: await writer.commit('1/1'); - const TEST_PARAMS = { group_id: 'group1' }; - const checkpoint = await bucketStorage.getCheckpoint(); - const parameters1 = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(MYBUCKET_1, [1152921504606846976n]) - ]); - expect(parameters1).toEqual([TEST_PARAMS]); + const n1 = 1152921504606846976n; + const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { n1 } }), {}); + + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + const buckets = await querier.queryDynamicBucketDescriptions({ + getParameterSets: async (lookups) => { + expect(lookups.map((l) => l.indexKey)).toEqual([[n1]]); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ group_id: 'group1' }]); + return parameter_sets; + } + }); + + expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'mybucket["group1"]').bucket]); }); test('save and load parameters with workspaceId', async () => { @@ -366,7 +413,7 @@ bucket_definitions: const buckets = await querier.queryDynamicBucketDescriptions({ async getParameterSets(lookups) { - expect(lookups).toEqual([ScopedParameterLookup.direct(parameterLookupScope('by_workspace', '1'), ['u1'])]); + expect(lookups.map((l) => l.indexKey)).toEqual([['u1']]); const parameter_sets = await checkpoint.getParameterSets(lookups); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }]); @@ -446,7 +493,7 @@ bucket_definitions: const buckets = await querier.queryDynamicBucketDescriptions({ async getParameterSets(lookups) { - expect(lookups).toEqual([ScopedParameterLookup.direct(parameterLookupScope('by_public_workspace', '1'), [])]); + expect(lookups.map((l) => l.indexKey)).toEqual([[]]); const parameter_sets = await checkpoint.getParameterSets(lookups); parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); @@ -560,10 +607,8 @@ bucket_definitions: } }) ).map((e) => e.bucket); - expect(foundLookups).toEqual([ - ScopedParameterLookup.direct(parameterLookupScope('by_workspace', '1'), []), - ScopedParameterLookup.direct(parameterLookupScope('by_workspace', '2'), ['u1']) - ]); + // Not testing the scope anymore - the exact format depends on storage version + expect(foundLookups.map((l) => l.indexKey)).toEqual([[], ['u1']]); parameter_sets.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); expect(parameter_sets).toEqual([{ workspace_id: 'workspace1' }, { workspace_id: 'workspace3' }]); @@ -591,6 +636,7 @@ bucket_definitions: ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -611,9 +657,19 @@ bucket_definitions: await writer.flush(); const checkpoint = await bucketStorage.getCheckpoint(); + const parameters = new RequestParameters(new JwtPayload({ sub: 'user1' }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['user1']]); - const parameters = await checkpoint.getParameterSets([ScopedParameterLookup.direct(MYBUCKET_1, ['user1'])]); - expect(parameters).toEqual([]); + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([]); + return parameter_sets; + } + }); + expect(buckets).toEqual([]); }); test('invalidate cached parsed sync rules', async () => { @@ -671,6 +727,7 @@ streams: `) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -688,12 +745,36 @@ streams: await writer.commit('1/1'); const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct(parameterLookupScope('lookup', '0'), ['baz']) - ]); - expect(parameters).toEqual([ + const parameters = new RequestParameters(new JwtPayload({ sub: 'baz' }), {}); + const querier = sync_rules.getBucketParameterQuerier({ + ...test_utils.querierOptions(parameters), + streams: { + stream: [ + { + parameters: null, + opaque_id: 123 + } + ] + } + }).querier; + + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['baz']]); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ '0': 'bar' }]); + return parameter_sets; + } + }); + console.log('whatabuckets', buckets); + expect(buckets).toHaveLength(1); + expect(buckets).toMatchObject([ { - '0': 'bar' + bucket: expect.stringMatching(/stream.*\["bar"\]$/), + definition: 'stream', + inclusion_reasons: [{ subscription: 123 }], + priority: 3 } ]); }); diff --git a/packages/sync-rules/src/BucketParameterQuerier.ts b/packages/sync-rules/src/BucketParameterQuerier.ts index e49a22967..b3a676999 100644 --- a/packages/sync-rules/src/BucketParameterQuerier.ts +++ b/packages/sync-rules/src/BucketParameterQuerier.ts @@ -120,6 +120,11 @@ export class ScopedParameterLookup { return (this.#cachedSerializedForm ??= JSONBig.stringify(this.values)); } + /** + * Index id. + * + * This depends on the lookup being constructed with lookupName = indexId, and queryId = ''. + */ get indexId(): string { const indexId = this.values[0]; // TODO: Consider restructuring so that these values aren't present at all @@ -131,6 +136,15 @@ export class ScopedParameterLookup { return indexId; } + /** + * Returns the "key" portion of the lookup values. + * + * This is this.values, excluding the first "lookupName" and "queryId". + */ + get indexKey(): SqliteJsonValue[] { + return this.values.slice(2); + } + static normalized(scope: ParameterLookupScope, lookup: UnscopedParameterLookup): ScopedParameterLookup { return new ScopedParameterLookup(scope.source, [scope.lookupName, scope.queryId, ...lookup.lookupValues]); } diff --git a/packages/sync-rules/src/HydrationState.ts b/packages/sync-rules/src/HydrationState.ts index 996de8056..6e07f0a38 100644 --- a/packages/sync-rules/src/HydrationState.ts +++ b/packages/sync-rules/src/HydrationState.ts @@ -8,7 +8,11 @@ export interface BucketDataScope { } export interface ParameterLookupScope { - /** The lookup name + queryid is used to reference the parameter lookup record. */ + /** + * The lookup name + queryid is used to reference the parameter lookup record. + * + * In newer storage versions, lookupName = indexId, and queryId = ''. + */ lookupName: string; queryId: string; /** Source used to generate parameter lookups. */ From 6051c05c3fe5f88394167ff4371d6711c1938f6b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 15:42:28 +0200 Subject: [PATCH 35/93] Further test fix. --- .../test/src/storage_sync.test.ts | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 27008f85d..8eab9278c 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -1,10 +1,10 @@ -import { deserializeParameterLookup, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { RequestParameters } from '@powersync/service-sync-rules'; import { describe, expect, test } from 'vitest'; -import { ScopedParameterLookup } from '@powersync/service-sync-rules'; -import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; -import { CurrentBucketV3, CurrentDataDocumentV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; +import { CurrentBucketV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) { register.registerSyncTests(storageConfig.factory, { @@ -160,14 +160,15 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor bucket_definitions: global: parameters: - - SELECT owner_id FROM "%" WHERE id = token_parameters.id + - SELECT owner_id FROM test WHERE id = token_parameters.test data: - - SELECT id, description, owner_id FROM "%" + - SELECT id, description, owner_id FROM test WHERE id = bucket.owner_id `, { storageVersion } ) ); const bucketStorage = factory.getInstance(syncRules); + const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); @@ -181,13 +182,23 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor }, afterReplicaId: test_utils.rid('shape-check') }); + await writer.markAllSnapshotDone('1/1'); await writer.commit('1/1'); const checkpoint = await bucketStorage.getCheckpoint(); - const parameters = await checkpoint.getParameterSets([ - ScopedParameterLookup.direct({ lookupName: 'global', queryId: '1', source: null as any }, ['shape-check']) - ]); - expect(parameters).toEqual([{ owner_id: 'user-1' }]); + const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {}); + const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; + const buckets = await querier.queryDynamicBucketDescriptions({ + async getParameterSets(lookups) { + expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]); + expect(lookups[0].indexId).toEqual('1'); + + const parameter_sets = await checkpoint.getParameterSets(lookups); + expect(parameter_sets).toEqual([{ owner_id: 'user-1' }]); + return parameter_sets; + } + }); + expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'global["user-1"]').bucket]); const mongoFactory = factory as MongoBucketStorage; const currentDataCollections = await mongoFactory.db.listSourceRecordCollections(syncRules.id); From 41c59b68aeaa49813de8935869e6cf266313df2f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 24 Mar 2026 15:51:36 +0200 Subject: [PATCH 36/93] Document collection structure. --- docs/data-ownership.md | 70 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 docs/data-ownership.md diff --git a/docs/data-ownership.md b/docs/data-ownership.md new file mode 100644 index 000000000..c5e4e3ca7 --- /dev/null +++ b/docs/data-ownership.md @@ -0,0 +1,70 @@ +# Storage version 3 - Data structure and "ownership" + +## Replication stream + +A replication stream represents one conceptual replication "job": + +1. One logical replication stream on the client in Postgres. +2. One change stream in MongoDB. +3. Generally, one entity creating checkpoints from a source database stream. + +This does not refer to concurrency - we may add concurrency in each of these streams at a later point, which may use multiple underlying database streams. Instead, it just refers to conceptually having one replication job, advancing checkpoints one at a time. + +Right now, each "sync config version", or `sync_rules` document, is one replication stream. + +For incremental reprocessing, this will change so that multiple sync config versions can be processed by the same stream. + +It is possible to have multiple replication streams running concurrently, for example when: + +1. Incremental reprocessing is not used, so nothing is shared between the sync config versions. +2. Changing storage versions - each replication stream can only handle one storage version at a time. + +## source_table + +Belongs to a replication stream. + +Scope: `source_table_${stream_id}` + +[FUTURE CHANGE] May have multiple copies per table per stream, especially when adding definitions. + +[FUTURE CHANGE] We can remove a source definition from a source table, but never add one. + +## source_records (previously current_data) + +Owned by source table. + +Scope: `source_records_${stream_id}_${source_table_id}` + +The `_id` field, is the source row id. Unlike V1 storage model, this does not include `g` (group_id) or `t` (table id), since those are already encapsulated in the collection name. + +When a table is dropped, we first create relevant REMOVE operations, then drop the relevant current_data collection. + +[FUTURE CHANGE] If a _definition_ using a source table is removed: + +1. We remove the bucket_data (drop the collection - see below). +2. We _don't_ update the source-records collection - stale records will remain. (Purely because this would be a slow operation, without gaining much) +3. When re-processing a source record, we then check for orphaned references. + +When all definitions for a source table is removed, we remove the drop the corresponding source_records collection. + +## bucket_data + +Owned by replication stream. + +Scoped by definition. + +Scope: `bucket_data_${stream_id}_${definition_id}` + +[FUTURE CHANGE] collection must be dropped when the definition is removed. + +## parameter_index (previously bucket_parameters) + +Owned by replication stream. + +Scoped by definition. + +Scope: `parameter_index_${stream_id}_${index_id}` + +_Also_ indexed by compound `key`, which includes {t: source_table_id, k: source_record_key} + +The `lookup` array drops the first two fields compared to V1 lookups (lookupName and queryId), since those are encapsulated in `index_id` in the collection name. In-memory, we use lookupName = indexId, queryId = '' (may change in the future). From fed52e312660d5be4b9fdbdf048bd587b12a6364 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Mon, 30 Mar 2026 13:39:52 +0200 Subject: [PATCH 37/93] Add some comments. --- .../src/storage/implementation/models.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 8b0d5aaa8..f54b22b3b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -334,8 +334,18 @@ export interface SyncRuleDocument { content: string; serialized_plan?: SerializedSyncPlan | null; + + /** + * Required for V3+ storage. + */ rule_mapping?: { + /** + * Map of uniqueName -> id, unique per replication stream. + */ definitions: Record; + /** + * Map of (lookupName, queryId) -> id, unique per replication stream. + */ parameter_indexes: Record; }; From f335b1d1598ab55c8e9218705e73cc6a567d8ddd Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 11:17:20 +0200 Subject: [PATCH 38/93] Rename. --- .../implementation/MongoParameterCompactor.ts | 4 ++-- .../implementation/MongoSyncBucketStorage.ts | 14 +++++++------- .../src/storage/implementation/PersistedBatchV1.ts | 2 +- .../src/storage/implementation/PersistedBatchV3.ts | 2 +- .../src/storage/implementation/db.ts | 10 +++++----- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts index cc9251ce4..b716c9f34 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts @@ -30,11 +30,11 @@ export class MongoParameterCompactor { } private async compactV1() { - await this.compactCollection(this.db.v1_bucket_parameters); + await this.compactCollection(this.db.parameterIndexV1); } private async compactV3() { - for (const collection of await this.db.listBucketParameterCollectionsV3(this.group_id)) { + for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { await this.compactCollection(collection); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 977d3479c..1af2a9c80 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -195,7 +195,7 @@ export class MongoSyncBucketStorage }); } for (let indexId of mapping.allParameterIndexIds()) { - await this.db.bucket_parameters_v3(this.group_id, indexId).createIndex( + await this.db.parameterIndexV3(this.group_id, indexId).createIndex( { lookup: 1, key: 1, @@ -420,7 +420,7 @@ export class MongoSyncBucketStorage // but could not do the same using $group. // For now, just rely on compacting to remove extraneous data. // For a description of the data format, see the `/docs/parameters-lookups.md` file. - const rows = await this.db.v1_bucket_parameters + const rows = await this.db.parameterIndexV1 .aggregate( [ { @@ -483,7 +483,7 @@ export class MongoSyncBucketStorage pipeline: mongo.Document[]; } => { const indexId = lookup.indexId; - const collection = this.db.bucket_parameters_v3(this.group_id, indexId); + const collection = this.db.parameterIndexV3(this.group_id, indexId); const lookupFilter = serializeParameterLookupV3(lookup); return { collection, @@ -975,11 +975,11 @@ export class MongoSyncBucketStorage ); } if (this.db.storageConfig.incrementalReprocessing) { - for (const collection of await this.db.listBucketParameterCollectionsV3(this.group_id)) { + for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { await collection.drop(); } } else { - await this.db.v1_bucket_parameters.deleteMany( + await this.db.parameterIndexV1.deleteMany( { 'key.g': this.group_id }, @@ -1319,7 +1319,7 @@ export class MongoSyncBucketStorage options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; - const parameterUpdates = await this.db.v1_bucket_parameters + const parameterUpdates = await this.db.parameterIndexV1 .find( { _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint }, @@ -1354,7 +1354,7 @@ export class MongoSyncBucketStorage const parameterUpdates: { lookup: bson.Binary; indexId: string }[] = []; // FIXME: Optimize performance for many collections - for (const collection of await this.db.listBucketParameterCollectionsV3(this.group_id)) { + for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { if (parameterUpdates.length > limit) { break; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index 8f60145d6..7e900b0b8 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -215,7 +215,7 @@ export class PersistedBatchV1 extends PersistedBatch { } protected async flushBucketParameters(session: mongo.ClientSession) { - await this.db.v1_bucket_parameters.bulkWrite( + await this.db.parameterIndexV1.bulkWrite( this.bucketParameters.map((document) => ({ insertOne: { document: taggedBucketParameterDocumentToV1(document) diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 3a18f2b0d..20d6033c4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -266,7 +266,7 @@ export class PersistedBatchV3 extends PersistedBatch { } for (const [indexId, documents] of operationsByIndex.entries()) { - await this.db.bucket_parameters_v3(this.group_id, indexId).bulkWrite( + await this.db.parameterIndexV3(this.group_id, indexId).bulkWrite( documents.map((document) => ({ insertOne: { document: taggedBucketParameterDocumentToV3(document) diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 20751946a..faee35130 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -413,7 +413,7 @@ export class VersionedPowerSyncMongo { return this.#upstream.listBucketDataCollectionsV3(groupId); } - get v1_bucket_parameters() { + get parameterIndexV1() { if (this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'bucket_parameters collection should not be used when incrementalReprocessing is enabled' @@ -422,22 +422,22 @@ export class VersionedPowerSyncMongo { return this.#upstream.bucket_parameters; } - bucket_parameters_v3(groupId: number, indexId: ParameterIndexId) { + parameterIndexV3(replicationStreamId: number, indexId: ParameterIndexId) { if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' ); } - return this.#upstream.parameterIndexV3(groupId, indexId); + return this.#upstream.parameterIndexV3(replicationStreamId, indexId); } - listBucketParameterCollectionsV3(groupId?: number) { + listParameterIndexCollectionsV3(replicationStreamId?: number) { if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' ); } - return this.#upstream.listParameterIndexCollectionsV3(groupId); + return this.#upstream.listParameterIndexCollectionsV3(replicationStreamId); } get op_id_sequence() { From a6619f7c3777e3329319c9c0fdfd7a337b7cdbab Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 11:32:25 +0200 Subject: [PATCH 39/93] Use $unionWith to find parameter index changes. --- .../implementation/MongoSyncBucketStorage.ts | 69 ++++++++++++------- .../src/storage/implementation/db.ts | 31 ++++++--- .../test/src/storage_sync.test.ts | 54 +++++++++++++++ 3 files changed, 119 insertions(+), 35 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 1af2a9c80..5290afdfe 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -976,7 +976,7 @@ export class MongoSyncBucketStorage } if (this.db.storageConfig.incrementalReprocessing) { for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { - await collection.drop(); + await collection.collection.drop(); } } else { await this.db.parameterIndexV1.deleteMany( @@ -1351,33 +1351,52 @@ export class MongoSyncBucketStorage options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; - const parameterUpdates: { lookup: bson.Binary; indexId: string }[] = []; - - // FIXME: Optimize performance for many collections - for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { - if (parameterUpdates.length > limit) { - break; + const collections = await this.db.listParameterIndexCollectionsV3(this.group_id); + if (collections.length == 0) { + return { + invalidateParameterBuckets: false, + updatedParameterLookups: new Set() + }; + } + const checkpointFilter = { + _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint } + }; + const collectionPrefix = `parameter_index_${this.group_id}_`; + const pipelineForCollection = (indexId: string) => [ + { + $match: checkpointFilter + }, + { + $project: { + _id: 0, + lookup: 1, + indexId: { $literal: indexId } + } } - - const remaining = limit + 1 - parameterUpdates.length; - const indexId = collection.collectionName.slice(`parameter_index_${this.group_id}_`.length); - const updates = await collection - .find( - { - _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint } - }, + ]; + const [firstCollection, ...remainingCollections] = collections; + const parameterUpdates = await firstCollection.collection + .aggregate<{ lookup: bson.Binary; indexId: string }>( + [ + ...pipelineForCollection(firstCollection.indexId), + ...remainingCollections.map((collection) => { + return { + $unionWith: { + coll: collection.collection.collectionName, + pipeline: pipelineForCollection(collection.indexId) + } + }; + }), { - projection: { - lookup: 1 - }, - limit: remaining, - batchSize: remaining + 1, - singleBatch: true + $limit: limit + 1 } - ) - .toArray(); - parameterUpdates.push(...updates.map((update) => ({ ...update, indexId }))); - } + ], + { + batchSize: limit + 2, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) + .toArray(); const invalidateParameterUpdates = parameterUpdates.length > limit; diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index faee35130..66efe74a9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -108,15 +108,12 @@ export class PowerSyncMongo { } /** - * List parameter index collections. + * List all parameter index collections across all replication streams. * - * @param replicationStreamId null only to list all collections in the db for clearing - * @returns + * Primarily used to clear the db. */ - async listParameterIndexCollectionsV3( - replicationStreamId?: number - ): Promise[]> { - const prefix = replicationStreamId == null ? `parameter_index_` : `parameter_index_${replicationStreamId}_`; + async listAllParameterIndexCollectionsV3(): Promise[]> { + const prefix = `parameter_index_`; const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); return collections @@ -183,7 +180,7 @@ export class PowerSyncMongo { await collection.drop(); } await this.bucket_parameters.deleteMany({}); - for (const collection of await this.listParameterIndexCollectionsV3()) { + for (const collection of await this.listAllParameterIndexCollectionsV3()) { await collection.drop(); } await this.op_id_sequence.deleteMany({}); @@ -431,13 +428,27 @@ export class VersionedPowerSyncMongo { return this.#upstream.parameterIndexV3(replicationStreamId, indexId); } - listParameterIndexCollectionsV3(replicationStreamId?: number) { + /** + * List parameter index collections for a specific replication stream. + */ + async listParameterIndexCollectionsV3( + replicationStreamId: number + ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]> { if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' ); } - return this.#upstream.listParameterIndexCollectionsV3(replicationStreamId); + + const prefix = `parameter_index_${replicationStreamId}_`; + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => ({ + collection: this.db.collection(collection.name), + indexId: collection.name.slice(prefix.length) + })); } get op_id_sequence() { diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 8eab9278c..2e2c77120 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -1,8 +1,10 @@ import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { JSONBig } from '@powersync/service-jsonbig'; import { RequestParameters } from '@powersync/service-sync-rules'; import { describe, expect, test } from 'vitest'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; +import { MongoSyncBucketStorage } from '../../src/storage/implementation/MongoSyncBucketStorage.js'; import { CurrentBucketV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -222,6 +224,58 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const parameterEntry = await mongoFactory.db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({}); expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']); }); + + test.runIf(storageVersion >= 3)( + 'loads parameter checkpoint changes across all v3 parameter index collections', + async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + by_owner: + parameters: + - SELECT owner_id FROM test WHERE id = token_parameters.owner_lookup + data: + - SELECT id, owner_id FROM test WHERE owner_id = bucket.owner_id + by_category: + parameters: + - SELECT category_id FROM test WHERE id = token_parameters.category_lookup + data: + - SELECT id, category_id FROM test WHERE category_id = bucket.category_id + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const previousCheckpoint = await bucketStorage.getCheckpoint(); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'shape-check', + owner_id: 'user-1', + category_id: 'cat-1' + }, + afterReplicaId: test_utils.rid('shape-check') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + + const nextCheckpoint = await bucketStorage.getCheckpoint(); + const changes = await bucketStorage.getCheckpointChanges({ + lastCheckpoint: previousCheckpoint, + nextCheckpoint + }); + + expect(changes.invalidateParameterBuckets).toBe(false); + expect(changes.updatedParameterLookups).toEqual(new Set(['["1","","shape-check"]', '["2","","shape-check"]'])); + } + ); } describe('sync - mongodb', () => { From 737de5154e0b152ac06de767160bb9c76c72c531 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 11:57:17 +0200 Subject: [PATCH 40/93] Fix current_data / source_records structure. --- .../src/storage/MongoBucketStorage.ts | 6 +- .../implementation/MongoBucketBatch.ts | 2 +- .../implementation/MongoParameterCompactor.ts | 2 +- .../implementation/MongoSyncBucketStorage.ts | 4 +- .../implementation/PersistedBatchV1.ts | 5 +- .../implementation/PersistedBatchV3.ts | 2 +- .../implementation/SourceRecordStoreV1.ts | 6 +- .../implementation/SourceRecordStoreV3.ts | 8 +- .../src/storage/implementation/db.ts | 95 ++++++++----------- .../test/src/storage_sync.test.ts | 43 ++++++++- 10 files changed, 100 insertions(+), 73 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 9169a31bb..233ae65f6 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -345,8 +345,10 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ]) .toArray() .catch(ignoreNotExisting); + + // FIXME: Handle v1 metrics const v3_parameter_aggregates = await Promise.all( - (await this.db.listParameterIndexCollectionsV3()).map((collection) => + (await this.db.listAllParameterIndexCollectionsV3()).map((collection) => collection .aggregate([ { @@ -361,7 +363,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ); const source_record_aggregates = await Promise.all( - (await this.db.listSourceRecordCollections()).map((collection) => + (await this.db.listAllSourceRecordCollectionsV3()).map((collection) => collection .aggregate([ { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index a71e49ede..1f3fc958a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -949,7 +949,7 @@ export abstract class MongoBucketBatch if (this.db.storageConfig.incrementalReprocessing) { for (let table of sourceTables) { await this.db - .common_current_data(this.group_id, mongoTableId(table.id)) + .sourceRecordsV3(this.group_id, mongoTableId(table.id)) .drop() .catch((error) => { if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts index b716c9f34..ffa909ae7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts @@ -35,7 +35,7 @@ export class MongoParameterCompactor { private async compactV3() { for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { - await this.compactCollection(collection); + await this.compactCollection(collection.collection); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 5290afdfe..9eaebc939 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -73,7 +73,7 @@ export class MongoSyncBucketStorage extends BaseObserver implements storage.SyncRulesBucketStorage { - private readonly db: VersionedPowerSyncMongo; + readonly db: VersionedPowerSyncMongo; readonly checksums: MongoChecksums; private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined; @@ -987,7 +987,7 @@ export class MongoSyncBucketStorage ); } - for (const collection of await this.db.listCommonCurrentDataCollections(this.group_id)) { + for (const collection of await this.db.listSourceRecordCollectionsV3(this.group_id)) { await collection.drop(); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index 7e900b0b8..ee82caf4f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -187,8 +187,7 @@ export class PersistedBatchV1 extends PersistedBatch { data: values.data ?? EMPTY_DATA, buckets, lookups - }, - $unset: { pending_delete: 1 } + } }, upsert: true } @@ -246,7 +245,7 @@ export class PersistedBatchV1 extends PersistedBatch { if (sourceTableId == null) { throw new ReplicationAssertionError('Missing source table id for current_data bulkWrite'); } - await this.db.v1_current_data(this.group_id, sourceTableId).bulkWrite(operations, { + await this.db.sourceRecordsV1(this.group_id, sourceTableId).bulkWrite(operations, { session, ordered: true }); diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 20d6033c4..57b342c9d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -291,7 +291,7 @@ export class PersistedBatchV3 extends PersistedBatch { for (const operations of operationsBySourceTable.values()) { const sourceTableId = operations[0]!.sourceTableId; - await this.db.v3_current_data(this.group_id, sourceTableId).bulkWrite( + await this.db.sourceRecordsV3(this.group_id, sourceTableId).bulkWrite( operations.map((entry) => entry.operation), { session, diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts index 93d3d68e7..cca3ada8c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts @@ -73,7 +73,7 @@ export class SourceRecordStoreV1 implements SourceRecordStore { const sizes = new Map(); for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { const sizeCursor: mongo.AggregationCursor = this.db - .v1_current_data(this.groupId, sourceTableId) + .sourceRecordsV1(this.groupId, sourceTableId) .aggregate( [ { @@ -107,7 +107,7 @@ export class SourceRecordStoreV1 implements SourceRecordStore { const documents = new Map(); const projection = idsOnly ? { _id: 1 } : undefined; for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { - const cursor = this.db.v1_current_data(this.groupId, sourceTableId).find( + const cursor = this.db.sourceRecordsV1(this.groupId, sourceTableId).find( { _id: { $in: replicaIds.map((replicaId) => this.createId(sourceTableId, replicaId) as SourceKey) @@ -134,7 +134,7 @@ export class SourceRecordStoreV1 implements SourceRecordStore { sourceTableId: bson.ObjectId, limit: number ): Promise { - const cursor = this.db.v1_current_data(this.groupId, sourceTableId).find( + const cursor = this.db.sourceRecordsV1(this.groupId, sourceTableId).find( { _id: idPrefixFilter({ g: this.groupId, t: sourceTableId }, ['k']), pending_delete: { $exists: false } diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts index e462db862..c357ef054 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts @@ -65,7 +65,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore { _id: { $in: replicaIds as any[] } } as unknown as mongo.Filter; const sizeCursor: mongo.AggregationCursor = this.db - .v3_current_data(this.groupId, sourceTableId) + .sourceRecordsV3(this.groupId, sourceTableId) .aggregate( [ { @@ -98,7 +98,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore { const filter = { _id: { $in: replicaIds as any[] } } as unknown as mongo.Filter; - const cursor = this.db.v3_current_data(this.groupId, sourceTableId).find(filter, { session, projection }); + const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find(filter, { session, projection }); for await (const doc of cursor.stream()) { const loaded = this.createLoadedDocument( sourceTableId, @@ -118,7 +118,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore { sourceTableId: bson.ObjectId, limit: number ): Promise { - const cursor = this.db.v3_current_data(this.groupId, sourceTableId).find( + const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find( { pending_delete: { $exists: false } }, @@ -139,7 +139,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore { async cleanup(lastCheckpoint: bigint, logger: Logger): Promise { let deletedCount = 0; - for (const collection of await this.db.listCommonCurrentDataCollections(this.groupId)) { + for (const collection of await this.db.listSourceRecordCollectionsV3(this.groupId)) { const result = await collection.deleteMany({ pending_delete: { $exists: true, $lte: lastCheckpoint } }); diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 66efe74a9..a8bccf92c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -36,11 +36,11 @@ export interface PowerSyncMongoOptions { export class PowerSyncMongo { readonly current_data: mongo.Collection; - readonly v3_current_data: mongo.Collection; readonly bucket_data: mongo.Collection; readonly bucket_parameters: mongo.Collection; readonly op_id_sequence: mongo.Collection; readonly sync_rules: mongo.Collection; + readonly source_tables: mongo.Collection; readonly custom_write_checkpoints: mongo.Collection; readonly write_checkpoints: mongo.Collection; readonly instance: mongo.Collection; @@ -61,11 +61,11 @@ export class PowerSyncMongo { this.db = db; this.current_data = db.collection('current_data'); - this.v3_current_data = db.collection('v3_current_data'); this.bucket_data = db.collection('bucket_data'); this.bucket_parameters = db.collection('bucket_parameters'); this.op_id_sequence = db.collection('op_id_sequence'); this.sync_rules = db.collection('sync_rules'); + this.source_tables = db.collection('source_tables'); this.custom_write_checkpoints = db.collection('custom_write_checkpoints'); this.write_checkpoints = db.collection('write_checkpoints'); this.instance = db.collection('instance'); @@ -108,17 +108,31 @@ export class PowerSyncMongo { } /** - * List all parameter index collections across all replication streams. - * - * Primarily used to clear the db. + * Not safe for user-provided prefix - only for hardcoded values. */ - async listAllParameterIndexCollectionsV3(): Promise[]> { - const prefix = `parameter_index_`; + private async collectionsByPrefix(prefix: string): Promise[]> { const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); return collections .filter((collection) => collection.name.startsWith(prefix)) - .map((collection) => this.db.collection(collection.name)); + .map((collection) => this.db.collection(collection.name)); + } + /** + * List all parameter index collections across all replication streams. + * + * Primarily used to clear the db. + */ + async listAllParameterIndexCollectionsV3(): Promise[]> { + return this.collectionsByPrefix(`parameter_index_`); + } + + /** + * List all parameter index collections across all replication streams. + * + * Primarily used to clear the db. + */ + async listAllSourceRecordCollectionsV3(): Promise[]> { + return this.collectionsByPrefix(`source_records_`); } sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) { @@ -129,24 +143,6 @@ export class PowerSyncMongo { return `source_table_${replicationStreamId}`; } - sourceRecords( - replicationStreamId: number, - sourceTableId: mongo.ObjectId - ): mongo.Collection { - return this.db.collection(this.sourceRecordsCollectionName(replicationStreamId, sourceTableId)); - } - - async listSourceRecordCollections( - replicationStreamId?: number - ): Promise[]> { - const prefix = replicationStreamId == null ? 'source_records_' : `source_records_${replicationStreamId}_`; - const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); - - return collections - .filter((collection) => collection.name.startsWith(prefix)) - .map((collection) => this.db.collection(collection.name)); - } - sourceTables(replicationStreamId: number): mongo.Collection { return this.db.collection(this.sourceTableCollectionName(replicationStreamId)); } @@ -171,8 +167,7 @@ export class PowerSyncMongo { */ async clear() { await this.current_data.deleteMany({}); - await this.v3_current_data.deleteMany({}); - for (const collection of await this.listSourceRecordCollections()) { + for (const collection of await this.listAllSourceRecordCollectionsV3()) { await collection.drop(); } await this.bucket_data.deleteMany({}); @@ -188,14 +183,7 @@ export class PowerSyncMongo { for (const collection of await this.listSourceTableCollections()) { await collection.drop(); } - for (const legacyName of ['source_tables', 'v3_source_tables']) { - await this.db.dropCollection(legacyName).catch((error) => { - if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { - return; - } - throw error; - }); - } + await this.source_tables.deleteMany({}); await this.write_checkpoints.deleteMany({}); await this.instance.deleteOne({}); await this.locks.deleteMany({}); @@ -310,38 +298,35 @@ export class VersionedPowerSyncMongo { this.storageConfig = storageConfig; } - /** - * Uses either `current_data` or `v3_current_data` collection based on the storage version. - * - * Use in places where it does not matter which version is used. - */ - common_current_data( - replicationStreamId: number, - sourceTableId: mongo.ObjectId - ): mongo.Collection { - return this.#upstream.sourceRecords(replicationStreamId, sourceTableId); - } - - v1_current_data(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + sourceRecordsV1(_replicationStreamId: number, _sourceTableId: mongo.ObjectId) { if (this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'current_data collection should not be used when incrementalReprocessing is enabled' ); } - return this.#upstream.sourceRecords(replicationStreamId, sourceTableId); + return this.#upstream.current_data; } - v3_current_data(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId) { if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'v3_current_data collection should not be used when incrementalReprocessing is disabled' ); } - return this.#upstream.sourceRecords(replicationStreamId, sourceTableId); + + const collectionName = `source_records_${replicationStreamId}_${sourceTableId.toHexString()}`; + return this.db.collection(collectionName); } - listCommonCurrentDataCollections(replicationStreamId?: number) { - return this.#upstream.listSourceRecordCollections(replicationStreamId); + async listSourceRecordCollectionsV3( + replicationStreamId: number + ): Promise[]> { + const prefix = `source_records_${replicationStreamId}_`; + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); } async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { @@ -350,7 +335,7 @@ export class VersionedPowerSyncMongo { 'source_records collection initialization should not be used when incrementalReprocessing is disabled' ); } - await this.#upstream.sourceRecords(replicationStreamId, sourceTableId).createIndex( + await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex( { pending_delete: 1 }, diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 2e2c77120..ad09d0935 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -203,7 +203,9 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'global["user-1"]').bucket]); const mongoFactory = factory as MongoBucketStorage; - const currentDataCollections = await mongoFactory.db.listSourceRecordCollections(syncRules.id); + const currentDataCollections = await (bucketStorage as MongoSyncBucketStorage).db.listSourceRecordCollectionsV3( + syncRules.id + ); const currentData = await currentDataCollections[0]?.findOne({}); const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined; expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/); @@ -225,6 +227,45 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']); }); + test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM test + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'shape-check', + description: 'shape' + }, + afterReplicaId: test_utils.rid('shape-check') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + + const mongoFactory = factory as MongoBucketStorage; + expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1); + + const sourceRecordCollections = await mongoFactory.db.db + .listCollections({ name: new RegExp(`^source_records_${syncRules.id}_`) }, { nameOnly: true }) + .toArray(); + expect(sourceRecordCollections).toEqual([]); + }); + test.runIf(storageVersion >= 3)( 'loads parameter checkpoint changes across all v3 parameter index collections', async () => { From 1adc0a1cfbfcacd3fbb4648de3d4a11b91fb1e41 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 12:03:20 +0200 Subject: [PATCH 41/93] Further fixes for v1 current_data. --- .../implementation/PersistedBatchV1.ts | 40 ++------- .../implementation/SourceRecordStoreV1.ts | 87 ++++++++----------- .../src/storage/implementation/db.ts | 2 +- 3 files changed, 43 insertions(+), 86 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index ee82caf4f..8d978c2d2 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -228,38 +228,20 @@ export class PersistedBatchV1 extends PersistedBatch { } protected async flushCurrentData(session: mongo.ClientSession) { - const operationsBySourceTable = new Map(); - for (const operation of this.currentData) { - const sourceTableId = this.getSourceTableIdHex(operation); - if (sourceTableId == null) { - throw new ReplicationAssertionError('Missing source table id for current_data operation'); - } - const existing = operationsBySourceTable.get(sourceTableId) ?? []; - existing.push(operation); - operationsBySourceTable.set(sourceTableId, existing); + if (this.currentData.length == 0) { + return; } - for (const operations of operationsBySourceTable.values()) { - const firstOperation = operations[0]!; - const sourceTableId = this.getSourceTableId(firstOperation); - if (sourceTableId == null) { - throw new ReplicationAssertionError('Missing source table id for current_data bulkWrite'); - } - await this.db.sourceRecordsV1(this.group_id, sourceTableId).bulkWrite(operations, { - session, - ordered: true - }); - } + await this.db.sourceRecordsV1.bulkWrite(this.currentData, { + session, + ordered: true + }); } protected resetCurrentData() { this.currentData = []; } - private getSourceTableIdHex(operation: mongo.AnyBulkWriteOperation): string | undefined { - return this.getSourceTableId(operation)?.toHexString(); - } - private currentDataId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey { return { g: this.group_id, @@ -267,14 +249,4 @@ export class PersistedBatchV1 extends PersistedBatch { k: replicaId }; } - - private getSourceTableId(operation: mongo.AnyBulkWriteOperation): bson.ObjectId | undefined { - if ('updateOne' in operation) { - return operation.updateOne.filter._id?.t; - } - if ('deleteOne' in operation) { - return operation.deleteOne.filter._id?.t; - } - return undefined; - } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts index cca3ada8c..224ba1d78 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts @@ -71,30 +71,27 @@ export class SourceRecordStoreV1 implements SourceRecordStore { async loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise> { const sizes = new Map(); - for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { - const sizeCursor: mongo.AggregationCursor = this.db - .sourceRecordsV1(this.groupId, sourceTableId) - .aggregate( - [ - { - $match: { - _id: { - $in: replicaIds.map((replicaId) => this.createId(sourceTableId, replicaId) as SourceKey) - } - } - }, - { - $project: { - _id: 1, - size: { $bsonSize: '$$ROOT' } + const sizeCursor: mongo.AggregationCursor = + this.db.sourceRecordsV1.aggregate( + [ + { + $match: { + _id: { + $in: entries.map((entry) => this.createId(entry.sourceTableId, entry.replicaId) as SourceKey) } } - ], - { session } - ); - for await (const doc of sizeCursor.stream()) { - sizes.set(cacheKey(sourceTableId, doc._id.k), doc.size); - } + }, + { + $project: { + _id: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ], + { session } + ); + for await (const doc of sizeCursor.stream()) { + sizes.set(cacheKey(doc._id.t, doc._id.k), doc.size); } return sizes; } @@ -106,25 +103,23 @@ export class SourceRecordStoreV1 implements SourceRecordStore { ): Promise> { const documents = new Map(); const projection = idsOnly ? { _id: 1 } : undefined; - for (const [sourceTableId, replicaIds] of this.groupEntries(entries)) { - const cursor = this.db.sourceRecordsV1(this.groupId, sourceTableId).find( - { - _id: { - $in: replicaIds.map((replicaId) => this.createId(sourceTableId, replicaId) as SourceKey) - } - }, - { session, projection } + const cursor = this.db.sourceRecordsV1.find( + { + _id: { + $in: entries.map((entry) => this.createId(entry.sourceTableId, entry.replicaId) as SourceKey) + } + }, + { session, projection } + ); + for await (const doc of cursor.stream()) { + const loaded = this.createLoadedDocument( + doc._id.t, + doc._id, + idsOnly ? null : doc.data, + idsOnly ? [] : doc.buckets, + idsOnly ? [] : doc.lookups ); - for await (const doc of cursor.stream()) { - const loaded = this.createLoadedDocument( - sourceTableId, - doc._id, - idsOnly ? null : doc.data, - idsOnly ? [] : doc.buckets, - idsOnly ? [] : doc.lookups - ); - documents.set(loaded.cacheKey, loaded); - } + documents.set(loaded.cacheKey, loaded); } return documents; } @@ -134,7 +129,7 @@ export class SourceRecordStoreV1 implements SourceRecordStore { sourceTableId: bson.ObjectId, limit: number ): Promise { - const cursor = this.db.sourceRecordsV1(this.groupId, sourceTableId).find( + const cursor = this.db.sourceRecordsV1.find( { _id: idPrefixFilter({ g: this.groupId, t: sourceTableId }, ['k']), pending_delete: { $exists: false } @@ -155,14 +150,4 @@ export class SourceRecordStoreV1 implements SourceRecordStore { } async cleanup(_lastCheckpoint: bigint, _logger: Logger): Promise {} - - private groupEntries(entries: SourceRecordLookupEntry[]): Map { - const grouped = new Map(); - for (const entry of entries) { - const existing = grouped.get(entry.sourceTableId) ?? []; - existing.push(entry.replicaId); - grouped.set(entry.sourceTableId, existing); - } - return grouped; - } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index a8bccf92c..60cf8c7fa 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -298,7 +298,7 @@ export class VersionedPowerSyncMongo { this.storageConfig = storageConfig; } - sourceRecordsV1(_replicationStreamId: number, _sourceTableId: mongo.ObjectId) { + get sourceRecordsV1() { if (this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'current_data collection should not be used when incrementalReprocessing is enabled' From 6208cdb0f7dcddb2ff4a1e084f8e79e24a58b97f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 12:07:13 +0200 Subject: [PATCH 42/93] Cleanup. --- .../src/storage/implementation/db.ts | 12 +++------ .../src/storage/implementation/models.ts | 1 - .../src/storage/current-data-table.ts | 26 ------------------- 3 files changed, 4 insertions(+), 35 deletions(-) delete mode 100644 modules/module-postgres-storage/src/storage/current-data-table.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 60cf8c7fa..e9cdd9444 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -2,7 +2,9 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { POWERSYNC_VERSION, storage } from '@powersync/service-core'; +import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { MongoStorageConfig } from '../../types/types.js'; +import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; import { BucketDataDocumentV1, BucketDataDocumentV3, @@ -11,7 +13,6 @@ import { BucketStateDocument, CheckpointEventDocument, ClientConnectionDocument, - CommonCurrentDataDocument, CommonSourceTableDocument, CurrentDataDocument, CurrentDataDocumentV3, @@ -19,13 +20,10 @@ import { IdSequenceDocument, InstanceDocument, SourceTableDocument, - SourceTableDocumentV3, StorageConfig, SyncRuleDocument, WriteCheckpointDocument } from './models.js'; -import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { BucketDefinitionId, BucketDefinitionMapping, ParameterIndexId } from './BucketDefinitionMapping.js'; export interface PowerSyncMongoOptions { /** @@ -318,15 +316,13 @@ export class VersionedPowerSyncMongo { return this.db.collection(collectionName); } - async listSourceRecordCollectionsV3( - replicationStreamId: number - ): Promise[]> { + async listSourceRecordCollectionsV3(replicationStreamId: number): Promise[]> { const prefix = `source_records_${replicationStreamId}_`; const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); return collections .filter((collection) => collection.name.startsWith(prefix)) - .map((collection) => this.db.collection(collection.name)); + .map((collection) => this.db.collection(collection.name)); } async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index f54b22b3b..ad7273680 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -431,7 +431,6 @@ export interface InstanceDocument { export interface ClientConnectionDocument extends event_types.ClientConnection {} export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id']; -export type CommonCurrentDataDocument = CurrentDataDocument | CurrentDataDocumentV3; export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3; export type CommonCurrentLookup = bson.Binary | RecordedLookupV3; export type CommonSourceTableDocument = SourceTableDocument | SourceTableDocumentV3; diff --git a/modules/module-postgres-storage/src/storage/current-data-table.ts b/modules/module-postgres-storage/src/storage/current-data-table.ts deleted file mode 100644 index 27f62ef14..000000000 --- a/modules/module-postgres-storage/src/storage/current-data-table.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { storage } from '@powersync/service-core'; - -export const V1_CURRENT_DATA_TABLE = 'current_data'; -export const V3_CURRENT_DATA_TABLE = 'v3_current_data'; - -/** - * The table used by a specific storage version for general current_data access. - */ -export function getCommonCurrentDataTable(storageConfig: storage.StorageVersionConfig) { - return storageConfig.softDeleteCurrentData ? V3_CURRENT_DATA_TABLE : V1_CURRENT_DATA_TABLE; -} - -export function getV1CurrentDataTable(storageConfig: storage.StorageVersionConfig) { - if (storageConfig.softDeleteCurrentData) { - throw new ServiceAssertionError('current_data table cannot be used when softDeleteCurrentData is enabled'); - } - return V1_CURRENT_DATA_TABLE; -} - -export function getV3CurrentDataTable(storageConfig: storage.StorageVersionConfig) { - if (!storageConfig.softDeleteCurrentData) { - throw new ServiceAssertionError('v3_current_data table cannot be used when softDeleteCurrentData is disabled'); - } - return V3_CURRENT_DATA_TABLE; -} From 9713f2d7a2d54bfa1276b42be5c80ddd7881a592 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 12:07:37 +0200 Subject: [PATCH 43/93] Fix type issue post merge. --- .../src/tests/register-data-storage-parameter-tests.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index 07208d495..88a7b0583 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -751,6 +751,7 @@ streams: streams: { stream: [ { + priorityOverride: null, parameters: null, opaque_id: 123 } From e76b1f424f062f9b86489327b2eea3f6e54c451c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 12:40:06 +0200 Subject: [PATCH 44/93] Rename postCommitCleanup. --- .../src/storage/implementation/MongoBucketBatch.ts | 2 +- .../src/storage/implementation/SourceRecordStore.ts | 2 +- .../src/storage/implementation/SourceRecordStoreV1.ts | 4 +++- .../src/storage/implementation/SourceRecordStoreV3.ts | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 1f3fc958a..88e1b207a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -815,7 +815,7 @@ export abstract class MongoBucketBatch this.persisted_op = null; this.last_checkpoint_lsn = lsn; if (newLastCheckpoint != null) { - await this.sourceRecordStore.cleanup(newLastCheckpoint, this.logger); + await this.sourceRecordStore.postCommitCleanup(newLastCheckpoint, this.logger); } } return { checkpointBlocked, checkpointCreated }; diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts index 1b0386414..a9fed1a36 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts @@ -45,5 +45,5 @@ export interface SourceRecordStore { sourceTableId: bson.ObjectId, limit: number ): Promise; - cleanup(lastCheckpoint: bigint, logger: Logger): Promise; + postCommitCleanup(lastCheckpoint: bigint, logger: Logger): Promise; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts index 224ba1d78..fcb80fd34 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts @@ -149,5 +149,7 @@ export class SourceRecordStoreV1 implements SourceRecordStore { ); } - async cleanup(_lastCheckpoint: bigint, _logger: Logger): Promise {} + async postCommitCleanup(_lastCheckpoint: bigint, _logger: Logger): Promise { + // No-op for V1. + } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts index c357ef054..4bd5cee57 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts @@ -137,7 +137,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore { ); } - async cleanup(lastCheckpoint: bigint, logger: Logger): Promise { + async postCommitCleanup(lastCheckpoint: bigint, logger: Logger): Promise { let deletedCount = 0; for (const collection of await this.db.listSourceRecordCollectionsV3(this.groupId)) { const result = await collection.deleteMany({ From 54349ed0c2d42579babeacd9599b13243d11a01c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 13:21:09 +0200 Subject: [PATCH 45/93] Track pending deletes & fix other source_table issues. --- .../implementation/MongoBucketBatch.ts | 6 +-- .../implementation/MongoSyncBucketStorage.ts | 41 +++++++++++------ .../implementation/PersistedBatchV3.ts | 36 +++++++++++++-- .../src/storage/implementation/db.ts | 46 ++++++++++++------- .../src/storage/implementation/models.ts | 8 +++- 5 files changed, 99 insertions(+), 38 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 88e1b207a..44a4ffafc 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -942,7 +942,7 @@ export abstract class MongoBucketBatch await this.withTransaction(async () => { for (let table of sourceTables) { - await this.db.source_tables(this.group_id).deleteOne({ _id: mongoTableId(table.id) }); + await this.db.commonSourceTables(this.group_id).deleteOne({ _id: mongoTableId(table.id) }); } }); @@ -1036,7 +1036,7 @@ export abstract class MongoBucketBatch copy.snapshotStatus = snapshotStatus; await this.withTransaction(async () => { - await this.db.source_tables(this.group_id).updateOne( + await this.db.commonSourceTables(this.group_id).updateOne( { _id: mongoTableId(table.id) }, { $set: { @@ -1091,7 +1091,7 @@ export abstract class MongoBucketBatch const ids = tables.map((table) => mongoTableId(table.id)); await this.withTransaction(async () => { - await this.db.source_tables(this.group_id).updateMany( + await this.db.commonSourceTables(this.group_id).updateMany( { _id: { $in: ids } }, { $set: { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 9eaebc939..387a98a2f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -267,15 +267,20 @@ export class MongoSyncBucketStorage const mapping = this.sync_rules.mapping; let result: storage.ResolveTableResult | null = null; let initializeSourceRecordsFor: bson.ObjectId | null = null; + + const baseId: Partial = this.db.storageConfig.incrementalReprocessing + ? {} + : { group_id }; await this.db.client.withSession(async (session) => { - const col = this.db.source_tables(group_id); + const col = this.db.commonSourceTables(group_id); let filter: Partial = { - group_id: group_id, + ...baseId, connection_id: connection_id, schema_name: schema, table_name: name, replica_id_columns2: normalizedReplicaIdColumns }; + if (objectId != null) { filter.relation_id = objectId; } @@ -292,7 +297,7 @@ export class MongoSyncBucketStorage }); const createDoc: CommonSourceTableDocument = { _id: candidateSourceTable.id as bson.ObjectId, - group_id: group_id, + ...(baseId as any), connection_id: connection_id, relation_id: objectId, schema_name: schema, @@ -353,7 +358,7 @@ export class MongoSyncBucketStorage const truncate = await col .find( { - group_id: group_id, + ...baseId, connection_id: connection_id, _id: { $ne: doc._id }, $or: truncateFilter @@ -998,15 +1003,25 @@ export class MongoSyncBucketStorage { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } ); - await this.db - .source_tables(this.group_id) - .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) - .catch((error) => { - if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { - return; - } - throw error; - }); + if (this.db.storageConfig.incrementalReprocessing) { + await this.db + .sourceTablesV3(this.group_id) + .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } else { + await this.db.commonSourceTables(this.group_id).deleteMany( + { + group_id: this.group_id + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ); + } + this.#storageInitialized = false; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 57b342c9d..ffecf8b3b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -1,6 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; -import { storage, utils } from '@powersync/service-core'; +import { InternalOpId, storage, utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; import * as bson from 'bson'; import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; @@ -17,12 +17,14 @@ import { CurrentDataDocumentV3, SourceTableKey, taggedBucketParameterDocumentToV3, - taggedBucketDataDocumentToV3 + taggedBucketDataDocumentToV3, + SourceTableDocumentV3 } from './models.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; export class PersistedBatchV3 extends PersistedBatch { currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation }[] = []; + sourceTablePendingDeletes = new Map(); saveBucketData(options: SaveBucketDataOptions) { const remaining_buckets = new Map(); @@ -167,7 +169,11 @@ export class PersistedBatchV3 extends PersistedBatch { this.currentSize += 50; } - softDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId, checkpointGreaterThan: bigint) { + softDeleteCurrentData( + sourceTableId: bson.ObjectId, + replicaId: storage.ReplicaId, + checkpointGreaterThan: InternalOpId + ) { this.currentData.push({ sourceTableId, operation: { @@ -185,6 +191,10 @@ export class PersistedBatchV3 extends PersistedBatch { } } }); + if (!this.sourceTablePendingDeletes.has(sourceTableId.toHexString())) { + this.sourceTablePendingDeletes.set(sourceTableId.toHexString(), checkpointGreaterThan); + } + this.currentSize += 50; } @@ -289,6 +299,25 @@ export class PersistedBatchV3 extends PersistedBatch { operationsBySourceTable.set(sourceTableId, existing); } + const sourceTableUpdates: mongo.AnyBulkWriteOperation[] = [ + ...this.sourceTablePendingDeletes.entries() + ].map(([key, value]) => { + return { + updateOne: { + filter: { _id: new bson.ObjectId(key) }, + update: { + $min: { + oldest_pending_delete: value + } + } + } + }; + }); + + if (sourceTableUpdates.length > 0) { + await this.db.sourceTablesV3(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false }); + } + for (const operations of operationsBySourceTable.values()) { const sourceTableId = operations[0]!.sourceTableId; await this.db.sourceRecordsV3(this.group_id, sourceTableId).bulkWrite( @@ -303,5 +332,6 @@ export class PersistedBatchV3 extends PersistedBatch { protected resetCurrentData() { this.currentData = []; + this.sourceTablePendingDeletes.clear(); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index e9cdd9444..205cae6cd 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -20,6 +20,7 @@ import { IdSequenceDocument, InstanceDocument, SourceTableDocument, + SourceTableDocumentV3, StorageConfig, SyncRuleDocument, WriteCheckpointDocument @@ -141,10 +142,6 @@ export class PowerSyncMongo { return `source_table_${replicationStreamId}`; } - sourceTables(replicationStreamId: number): mongo.Collection { - return this.db.collection(this.sourceTableCollectionName(replicationStreamId)); - } - async listSourceTableCollections( replicationStreamId?: number ): Promise[]> { @@ -342,22 +339,37 @@ export class VersionedPowerSyncMongo { ); } - source_tables(replicationStreamId: number): mongo.Collection { - return this.#upstream.sourceTables(replicationStreamId); + commonSourceTables(replicationStreamId: number): mongo.Collection { + if (this.storageConfig.incrementalReprocessing) { + return this.sourceTablesV3(replicationStreamId) as mongo.Collection; + } else { + return this.#upstream.source_tables as any as mongo.Collection; + } + } + + sourceTablesV3(replicationStreamId: number) { + if (!this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'source_tables v3 collection should not be used when incrementalReprocessing is disabled' + ); + } + return this.db.collection(this.#upstream.sourceTableCollectionName(replicationStreamId)); } async initializeStreamStorage(replicationStreamId: number) { - await this.source_tables(replicationStreamId).createIndex( - { - connection_id: 1, - schema_name: 1, - table_name: 1, - relation_id: 1 - }, - { - name: 'source_lookup' - } - ); + if (this.storageConfig.incrementalReprocessing) { + await this.sourceTablesV3(replicationStreamId).createIndex( + { + connection_id: 1, + schema_name: 1, + table_name: 1, + relation_id: 1 + }, + { + name: 'source_lookup' + } + ); + } } get bucket_data() { diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index ad7273680..0f3576bc6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -195,7 +195,6 @@ export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; export interface SourceTableDocument { _id: bson.ObjectId; - group_id: number; connection_id: number; relation_id: number | string | undefined; schema_name: string; @@ -206,9 +205,14 @@ export interface SourceTableDocument { snapshot_status: SourceTableDocumentSnapshotStatus | undefined; } +export interface SourceTableDocumentV1 extends SourceTableDocument { + group_id: number; +} + export interface SourceTableDocumentV3 extends SourceTableDocument { bucket_data_source_ids: BucketDefinitionId[]; parameter_lookup_source_ids: ParameterIndexId[]; + oldest_pending_delete?: InternalOpId | undefined; } export interface SourceTableDocumentSnapshotStatus { @@ -433,7 +437,7 @@ export interface ClientConnectionDocument extends event_types.ClientConnection { export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id']; export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3; export type CommonCurrentLookup = bson.Binary | RecordedLookupV3; -export type CommonSourceTableDocument = SourceTableDocument | SourceTableDocumentV3; +export type CommonSourceTableDocument = SourceTableDocumentV1 | SourceTableDocumentV3; export function isCurrentBucketV3(bucket: CommonCurrentBucket): bucket is CurrentBucketV3 { return 'def' in bucket; From 5287d28e37d4e4595e73b54ac0c828e111bb8a24 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 13:35:33 +0200 Subject: [PATCH 46/93] Only cleanup pending deletes for affected source tables. --- .../implementation/PersistedBatchV3.ts | 10 ++- .../implementation/SourceRecordStoreV3.ts | 43 +++++++++- .../src/storage/implementation/db.ts | 12 ++- .../src/storage/implementation/models.ts | 2 +- .../test/src/storage_sync.test.ts | 85 +++++++++++++++++++ 5 files changed, 144 insertions(+), 8 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index ffecf8b3b..95aec78bb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -191,8 +191,10 @@ export class PersistedBatchV3 extends PersistedBatch { } } }); - if (!this.sourceTablePendingDeletes.has(sourceTableId.toHexString())) { - this.sourceTablePendingDeletes.set(sourceTableId.toHexString(), checkpointGreaterThan); + const sourceTableKey = sourceTableId.toHexString(); + const existingPendingDelete = this.sourceTablePendingDeletes.get(sourceTableKey); + if (existingPendingDelete == null || checkpointGreaterThan > existingPendingDelete) { + this.sourceTablePendingDeletes.set(sourceTableKey, checkpointGreaterThan); } this.currentSize += 50; @@ -306,8 +308,8 @@ export class PersistedBatchV3 extends PersistedBatch { updateOne: { filter: { _id: new bson.ObjectId(key) }, update: { - $min: { - oldest_pending_delete: value + $max: { + latest_pending_delete: value } } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts index 4bd5cee57..33132450e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts @@ -6,7 +6,7 @@ import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules import { VersionedPowerSyncMongo } from './db.js'; import { cacheKey } from './OperationBatch.js'; import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from './SourceRecordStore.js'; -import { CurrentDataDocumentV3 } from './models.js'; +import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; @@ -138,12 +138,51 @@ export class SourceRecordStoreV3 implements SourceRecordStore { } async postCommitCleanup(lastCheckpoint: bigint, logger: Logger): Promise { + // This cleans up soft deletes in source_records collections. + // Since there may be a lot (100+) of these collections in some cases, we track which + // ones have dirty deletes in source_tables. + + const dirtySourceTables = await this.db + .sourceTablesV3(this.groupId) + .find( + { + latest_pending_delete: { $exists: true } + }, + { + projection: { _id: 1, latest_pending_delete: 1 } + } + ) + .toArray(); + let deletedCount = 0; - for (const collection of await this.db.listSourceRecordCollectionsV3(this.groupId)) { + const sourceTableUpdates: mongo.AnyBulkWriteOperation[] = []; + for (const sourceTable of dirtySourceTables) { + const collection = this.db.sourceRecordsV3(this.groupId, sourceTable._id); const result = await collection.deleteMany({ pending_delete: { $exists: true, $lte: lastCheckpoint } }); deletedCount += result.deletedCount; + + if (sourceTable.latest_pending_delete != null && sourceTable.latest_pending_delete <= lastCheckpoint) { + sourceTableUpdates.push({ + updateOne: { + filter: { + _id: sourceTable._id, + // If the source table received more writes in the meantime, this will filter it out + latest_pending_delete: sourceTable.latest_pending_delete + }, + update: { + $unset: { + latest_pending_delete: 1 + } + } + } + }); + } + } + + if (sourceTableUpdates.length > 0) { + await this.db.sourceTablesV3(this.groupId).bulkWrite(sourceTableUpdates, { ordered: false }); } if (deletedCount > 0) { logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`); diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 205cae6cd..d75173088 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -358,7 +358,8 @@ export class VersionedPowerSyncMongo { async initializeStreamStorage(replicationStreamId: number) { if (this.storageConfig.incrementalReprocessing) { - await this.sourceTablesV3(replicationStreamId).createIndex( + const sourceTables = this.sourceTablesV3(replicationStreamId); + await sourceTables.createIndex( { connection_id: 1, schema_name: 1, @@ -369,6 +370,15 @@ export class VersionedPowerSyncMongo { name: 'source_lookup' } ); + await sourceTables.createIndex( + { + latest_pending_delete: 1 + }, + { + partialFilterExpression: { latest_pending_delete: { $exists: true } }, + name: 'latest_pending_delete' + } + ); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 0f3576bc6..e3b507fd7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -212,7 +212,7 @@ export interface SourceTableDocumentV1 extends SourceTableDocument { export interface SourceTableDocumentV3 extends SourceTableDocument { bucket_data_source_ids: BucketDefinitionId[]; parameter_lookup_source_ids: ParameterIndexId[]; - oldest_pending_delete?: InternalOpId | undefined; + latest_pending_delete?: InternalOpId | undefined; } export interface SourceTableDocumentSnapshotStatus { diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index ad09d0935..75a5f459d 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -2,9 +2,11 @@ import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYam import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { JSONBig } from '@powersync/service-jsonbig'; import { RequestParameters } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/MongoSyncBucketStorage.js'; +import { SourceRecordStoreV3 } from '../../src/storage/implementation/SourceRecordStoreV3.js'; import { CurrentBucketV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -317,6 +319,89 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor expect(changes.updatedParameterLookups).toEqual(new Set(['["1","","shape-check"]', '["2","","shape-check"]'])); } ); + + test.runIf(storageVersion >= 3)('cleans pending deletes only for tracked v3 source tables', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM test + `, + { storageVersion } + ) + ); + + const mongoFactory = factory as MongoBucketStorage; + const bucketStorage = mongoFactory.getInstance(syncRules) as any; + const db = bucketStorage.db; + await db.initializeStreamStorage(syncRules.id); + + const sourceTableA = new bson.ObjectId(); + const sourceTableB = new bson.ObjectId(); + await db.sourceTablesV3(syncRules.id).insertMany([ + { + _id: sourceTableA, + connection_id: 1, + relation_id: 'a', + schema_name: 'public', + table_name: 'table_a', + replica_id_columns: null, + replica_id_columns2: [], + snapshot_done: true, + snapshot_status: undefined, + bucket_data_source_ids: [], + parameter_lookup_source_ids: [], + latest_pending_delete: 9n + }, + { + _id: sourceTableB, + connection_id: 1, + relation_id: 'b', + schema_name: 'public', + table_name: 'table_b', + replica_id_columns: null, + replica_id_columns2: [], + snapshot_done: true, + snapshot_status: undefined, + bucket_data_source_ids: [], + parameter_lookup_source_ids: [], + latest_pending_delete: 12n + } + ]); + + await db.sourceRecordsV3(syncRules.id, sourceTableA).insertMany([ + { _id: 'deleted-1', data: null, buckets: [], lookups: [], pending_delete: 5n }, + { _id: 'deleted-2', data: null, buckets: [], lookups: [], pending_delete: 9n }, + { _id: 'active', data: null, buckets: [], lookups: [] } + ]); + await db + .sourceRecordsV3(syncRules.id, sourceTableB) + .insertMany([{ _id: 'later-delete', data: null, buckets: [], lookups: [], pending_delete: 12n }]); + + const store = new SourceRecordStoreV3(db, syncRules.id, bucketStorage.sync_rules.mapping); + const logger = { info() {} } as any; + + await store.postCommitCleanup(6n, logger); + + expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 5n })).toBe(0); + expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 9n })).toBe(1); + expect(await db.sourceRecordsV3(syncRules.id, sourceTableB).countDocuments({ pending_delete: 12n })).toBe(1); + expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete).toBe(9n); + expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n); + + await store.postCommitCleanup(10n, logger); + + expect( + await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: { $exists: true } }) + ).toBe(0); + expect( + (await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete + ).toBeUndefined(); + expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n); + }); } describe('sync - mongodb', () => { From edb3f89830e16bc711735b70053e2f675bcbcfb4 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 13:43:49 +0200 Subject: [PATCH 47/93] Add timeout and retry for deletes. --- .../implementation/MongoSyncBucketStorage.ts | 43 ++++++++----------- .../implementation/SourceRecordStoreV3.ts | 31 +++++++++++-- .../module-mongodb-storage/src/utils/util.ts | 31 ++++++++++++- 3 files changed, 76 insertions(+), 29 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 387a98a2f..9812e2091 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -1,11 +1,6 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; -import { - BaseObserver, - logger, - ReplicationAbortedError, - ServiceAssertionError -} from '@powersync/lib-services-framework'; +import { BaseObserver, logger, ServiceAssertionError } from '@powersync/lib-services-framework'; import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, @@ -29,7 +24,13 @@ import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powers import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; -import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../utils/util.js'; +import { + idPrefixFilter, + mapOpEntry, + readSingleBatch, + retryOnMongoMaxTimeMSExpired, + setSessionSnapshotTime +} from '../../utils/util.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { VersionedPowerSyncMongo } from './db.js'; import { @@ -923,26 +924,18 @@ export class MongoSyncBucketStorage } async clear(options?: storage.ClearStorageOptions): Promise { - while (true) { - if (options?.signal?.aborted) { - throw new ReplicationAbortedError('Aborted clearing data', options.signal.reason); + await retryOnMongoMaxTimeMSExpired(() => this.clearIteration(), { + signal: options?.signal, + abortMessage: 'Aborted clearing data', + retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5, + onRetry: () => { + logger.info( + `${this.slot_name} Cleared batch of data in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` + ); } - try { - await this.clearIteration(); + }); - logger.info(`${this.slot_name} Done clearing data`); - return; - } catch (e: unknown) { - if (lib_mongo.isMongoServerError(e) && e.codeName == 'MaxTimeMSExpired') { - logger.info( - `${this.slot_name} Cleared batch of data in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` - ); - await timers.setTimeout(lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5); - } else { - throw e; - } - } - } + logger.info(`${this.slot_name} Done clearing data`); } private async clearIteration(): Promise { diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts index 33132450e..6baa9d795 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts @@ -1,8 +1,10 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import * as bson from 'bson'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import { retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; import { VersionedPowerSyncMongo } from './db.js'; import { cacheKey } from './OperationBatch.js'; import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from './SourceRecordStore.js'; @@ -158,9 +160,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore { const sourceTableUpdates: mongo.AnyBulkWriteOperation[] = []; for (const sourceTable of dirtySourceTables) { const collection = this.db.sourceRecordsV3(this.groupId, sourceTable._id); - const result = await collection.deleteMany({ - pending_delete: { $exists: true, $lte: lastCheckpoint } - }); + const result = await this.deletePendingDeletes(collection, sourceTable._id, lastCheckpoint, logger); deletedCount += result.deletedCount; if (sourceTable.latest_pending_delete != null && sourceTable.latest_pending_delete <= lastCheckpoint) { @@ -189,6 +189,31 @@ export class SourceRecordStoreV3 implements SourceRecordStore { } } + private async deletePendingDeletes( + collection: mongo.Collection, + sourceTableId: bson.ObjectId, + lastCheckpoint: bigint, + logger: Logger + ) { + return retryOnMongoMaxTimeMSExpired( + () => + collection.deleteMany( + { + pending_delete: { $exists: true, $lte: lastCheckpoint } + }, + { + maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS + } + ), + { + retryDelayMs: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS / 5, + onRetry: (n: number) => { + logger.warn(`Cleared batch ${n} of pending deletes for source table ${sourceTableId}, continuing...`); + } + } + ); + } + private groupEntries(entries: SourceRecordLookupEntry[]): Map { const grouped = new Map(); for (const entry of entries) { diff --git a/modules/module-mongodb-storage/src/utils/util.ts b/modules/module-mongodb-storage/src/utils/util.ts index 06342cbb9..2e0a1cf9c 100644 --- a/modules/module-mongodb-storage/src/utils/util.ts +++ b/modules/module-mongodb-storage/src/utils/util.ts @@ -1,9 +1,11 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; import * as bson from 'bson'; import * as crypto from 'crypto'; +import * as timers from 'node:timers/promises'; import * as uuid from 'uuid'; import { mongo } from '@powersync/lib-service-mongodb'; import { storage, utils } from '@powersync/service-core'; -import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework'; import { TaggedBucketDataDocument } from '../storage/implementation/models.js'; export function idPrefixFilter(prefix: Partial, rest: (keyof T)[]): mongo.Condition { @@ -129,6 +131,33 @@ export function setSessionSnapshotTime(session: mongo.ClientSession, time: bson. } } +export async function retryOnMongoMaxTimeMSExpired( + operation: () => Promise, + options: { + signal?: AbortSignal; + abortMessage?: string; + retryDelayMs: number; + onRetry?: (retryCount: number) => void; + } +): Promise { + let retryCount = 0; + while (true) { + if (options.signal?.aborted) { + throw new ReplicationAbortedError(options.abortMessage ?? 'Aborted MongoDB operation', options.signal.reason); + } + try { + return await operation(); + } catch (e) { + if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') { + throw e; + } + retryCount += 1; + options.onRetry?.(retryCount); + await timers.setTimeout(options.retryDelayMs); + } + } +} + export const createPaginatedConnectionQuery = async ( query: mongo.Filter, collection: mongo.Collection, From 3a0668f7e3467e01a0cbbcaf4c913ddae896e914 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 13:44:37 +0200 Subject: [PATCH 48/93] Increase clear timeout. --- libs/lib-mongodb/src/db/mongo.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/lib-mongodb/src/db/mongo.ts b/libs/lib-mongodb/src/db/mongo.ts index b57d833f7..21678ff51 100644 --- a/libs/lib-mongodb/src/db/mongo.ts +++ b/libs/lib-mongodb/src/db/mongo.ts @@ -31,11 +31,13 @@ export const MONGO_OPERATION_TIMEOUT_MS = 40_000; export const MONGO_CHECKSUM_TIMEOUT_MS = 50_000; /** - * Same as above, but specifically for clear operations. + * Same as MONGO_OPERATION_TIMEOUT_MS, but specifically for clear operations. * * These are retried when reaching the timeout. + * + * Used to be 5s. Increased to attempt to improve efficiency (deleted documents / scanned documents). */ -export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5_000; +export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = MONGO_OPERATION_TIMEOUT_MS; export interface MongoConnectionOptions { maxPoolSize?: number; From 9bf8cd21030fc5153d962cfd7ec18b9ed07201cd Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 13:48:29 +0200 Subject: [PATCH 49/93] Retry inner clear operations instead of the entire loop. --- .../implementation/MongoSyncBucketStorage.ts | 104 +++++++++++------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 9812e2091..5269abef6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -1,6 +1,11 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; -import { BaseObserver, logger, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { + BaseObserver, + logger, + ReplicationAbortedError, + ServiceAssertionError +} from '@powersync/lib-services-framework'; import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, @@ -924,23 +929,11 @@ export class MongoSyncBucketStorage } async clear(options?: storage.ClearStorageOptions): Promise { - await retryOnMongoMaxTimeMSExpired(() => this.clearIteration(), { - signal: options?.signal, - abortMessage: 'Aborted clearing data', - retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5, - onRetry: () => { - logger.info( - `${this.slot_name} Cleared batch of data in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` - ); - } - }); - - logger.info(`${this.slot_name} Done clearing data`); - } + const signal = options?.signal; - private async clearIteration(): Promise { - // Individual operations here may time out with the maxTimeMS option. - // It is expected to still make progress, and continue on the next try. + if (signal?.aborted) { + throw new ReplicationAbortedError('Aborted clearing data', signal.reason); + } await this.db.sync_rules.updateOne( { @@ -965,11 +958,16 @@ export class MongoSyncBucketStorage await collection.drop(); } } else { - await this.db.bucket_data.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['b', 'o']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + await this.clearDeleteMany( + 'bucket data', + () => + this.db.bucket_data.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['b', 'o']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal ); } if (this.db.storageConfig.incrementalReprocessing) { @@ -977,11 +975,16 @@ export class MongoSyncBucketStorage await collection.collection.drop(); } } else { - await this.db.parameterIndexV1.deleteMany( - { - 'key.g': this.group_id - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + await this.clearDeleteMany( + 'parameter index', + () => + this.db.parameterIndexV1.deleteMany( + { + 'key.g': this.group_id + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal ); } @@ -989,11 +992,16 @@ export class MongoSyncBucketStorage await collection.drop(); } - await this.db.bucket_state.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['b']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + await this.clearDeleteMany( + 'bucket state', + () => + this.db.bucket_state.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['b']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal ); if (this.db.storageConfig.incrementalReprocessing) { @@ -1007,17 +1015,39 @@ export class MongoSyncBucketStorage throw error; }); } else { - await this.db.commonSourceTables(this.group_id).deleteMany( - { - group_id: this.group_id - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + await this.clearDeleteMany( + 'source tables', + () => + this.db.commonSourceTables(this.group_id).deleteMany( + { + group_id: this.group_id + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal ); } this.#storageInitialized = false; } + private async clearDeleteMany( + label: string, + operation: () => Promise, + signal?: AbortSignal + ): Promise { + await retryOnMongoMaxTimeMSExpired(operation, { + signal, + abortMessage: 'Aborted clearing data', + retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5, + onRetry: () => { + logger.info( + `${this.slot_name} Cleared batch of ${label} in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` + ); + } + }); + } + async reportError(e: any): Promise { const message = String(e.message ?? 'Replication failure'); await this.db.sync_rules.updateOne( From 653477b48032c24edadce59cf8015d0fea79cef5 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 14:19:16 +0200 Subject: [PATCH 50/93] Split bucket_state collections. --- .../storage/implementation/MongoChecksums.ts | 232 ++++++++++++------ .../storage/implementation/MongoCompactor.ts | 175 +++++++++---- .../implementation/MongoSyncBucketStorage.ts | 90 ++++++- .../storage/implementation/PersistedBatch.ts | 76 +++++- .../implementation/PersistedBatchV1.ts | 4 +- .../implementation/PersistedBatchV3.ts | 4 +- .../src/storage/implementation/db.ts | 54 +++- .../src/storage/implementation/models.ts | 19 +- .../test/src/storage_compacting.test.ts | 68 +++-- 9 files changed, 537 insertions(+), 185 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index d8c0222b4..bd889c857 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -16,9 +16,22 @@ import { PartialOrFullChecksum } from '@powersync/service-core'; import { VersionedPowerSyncMongo } from './db.js'; -import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { BucketDefinitionId, BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { StorageConfig } from './models.js'; +export interface FetchPartialBucketChecksumV3 { + bucket: string; + definitionId: BucketDefinitionId; + start?: InternalOpId; + end: InternalOpId; +} + +export interface FetchPartialBucketChecksumByBucket { + bucket: string; + start?: InternalOpId; + end: InternalOpId; +} + /** * Checksum calculation options, primarily for tests. */ @@ -100,41 +113,7 @@ abstract class AbstractMongoChecksums { if (batch.length == 0) { return new Map(); } - - const preFilters: any[] = []; - for (let request of batch) { - if (request.start == null) { - preFilters.push({ - _id: { - g: this.group_id, - b: request.bucket - }, - 'compacted_state.op_id': { $exists: true, $lte: request.end } - }); - } - } - - const preStates = new Map(); - - if (preFilters.length > 0) { - // For un-cached bucket checksums, attempt to use the compacted state first. - const states = await this.db.bucket_state - .find({ - $or: preFilters - }) - .toArray(); - for (let state of states) { - const compactedState = state.compacted_state!; - preStates.set(state._id.b, { - opId: compactedState.op_id, - checksum: { - bucket: state._id.b, - checksum: Number(compactedState.checksum), - count: compactedState.count - } - }); - } - } + const preStates = await this.fetchPreStates(batch); const mappedRequests = batch.map((request) => { let start = request.start; @@ -205,15 +184,19 @@ abstract class AbstractMongoChecksums { */ protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise; - protected async computePartialChecksumsForCollection( - batch: FetchPartialBucketChecksum[], + protected abstract fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise>; + + protected async computePartialChecksumsForCollection( + batch: TRequest[], collection: mongo.Collection, - createFilter: (request: FetchPartialBucketChecksum) => any + createFilter: (request: TRequest) => any ): Promise { const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT; // Map requests by bucket. We adjust this as we get partial results. - let requests = new Map(); + let requests = new Map(); for (let request of batch) { requests.set(request.bucket, request); } @@ -292,10 +275,8 @@ abstract class AbstractMongoChecksums { limitReached = true; const req = requests.get(bucket); requests.set(bucket, { - bucket, - source: req!.source, - start: doc.last_op, - end: req!.end + ...req!, + start: doc.last_op }); } else { // All done for this bucket @@ -334,7 +315,9 @@ abstract class AbstractMongoChecksums { } class MongoChecksumsV1Impl extends AbstractMongoChecksums { - protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + async computePartialChecksumsDirectByBucket( + batch: FetchPartialBucketChecksumByBucket[] + ): Promise { return this.computePartialChecksumsForCollection( batch, this.db.bucket_data as unknown as mongo.Collection, @@ -354,6 +337,49 @@ class MongoChecksumsV1Impl extends AbstractMongoChecksums { }) ); } + + protected async fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise> { + const preFilters = batch + .filter((request) => request.start == null) + .map((request) => ({ + _id: { + g: this.group_id, + b: request.bucket + }, + 'compacted_state.op_id': { $exists: true, $lte: request.end } + })); + + const preStates = new Map(); + if (preFilters.length == 0) { + return preStates; + } + + const states = await this.db.bucketStateV1 + .find({ + $or: preFilters + }) + .toArray(); + + for (const state of states) { + const compactedState = state.compacted_state!; + preStates.set(state._id.b, { + opId: compactedState.op_id, + checksum: { + bucket: state._id.b, + checksum: Number(compactedState.checksum), + count: compactedState.count + } + }); + } + + return preStates; + } + + protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + return this.computePartialChecksumsDirectByBucket(batch); + } } class MongoChecksumsV3Impl extends AbstractMongoChecksums { @@ -366,21 +392,23 @@ class MongoChecksumsV3Impl extends AbstractMongoChecksums { super(db, group_id, options); } - protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + private normalizeBatch(batch: FetchPartialBucketChecksum[]): FetchPartialBucketChecksumV3[] { + return batch.map((request) => ({ + bucket: request.bucket, + definitionId: this.mapping.bucketSourceId(request.source), + start: request.start, + end: request.end + })); + } + + async computePartialChecksumsDirectByDefinition(batch: FetchPartialBucketChecksumV3[]): Promise { const results = new Map(); - const requestsByDefinition = new Map(); - const fallbackRequests: FetchPartialBucketChecksum[] = []; + const requestsByDefinition = new Map(); for (const request of batch) { - if (!isBucketSourceLike(request.source)) { - fallbackRequests.push(request); - continue; - } - - const definitionId = this.mapping.bucketSourceId(request.source); - const existing = requestsByDefinition.get(definitionId) ?? []; + const existing = requestsByDefinition.get(request.definitionId) ?? []; existing.push(request); - requestsByDefinition.set(definitionId, existing); + requestsByDefinition.set(request.definitionId, existing); } for (const [definitionId, requests] of requestsByDefinition.entries()) { @@ -394,23 +422,53 @@ class MongoChecksumsV3Impl extends AbstractMongoChecksums { } } - if (fallbackRequests.length > 0) { - const collections = await this.db.listBucketDataCollectionsV3(this.group_id); - for (const request of fallbackRequests) { - let merged: PartialOrFullChecksum | null = null; - for (const collection of collections) { - const groupResults = await this.computePartialChecksumsForCollection( - [request], - collection as unknown as mongo.Collection, - createV3BucketFilter - ); - merged = addPartialChecksums(request.bucket, merged, groupResults.get(request.bucket) ?? null); + return new Map( + batch.map((request) => [request.bucket, results.get(request.bucket) ?? emptyChecksumForRequest(request)]) + ); + } + + protected async fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise> { + const preFilters = this.normalizeBatch(batch) + .filter((request) => request.start == null) + .map((request) => ({ + _id: { + d: request.definitionId, + b: request.bucket + }, + 'compacted_state.op_id': { $exists: true, $lte: request.end } + })); + + const preStates = new Map(); + if (preFilters.length == 0) { + return preStates; + } + + const states = await this.db + .bucketStateV3(this.group_id) + .find({ + $or: preFilters + }) + .toArray(); + + for (const state of states) { + const compactedState = state.compacted_state!; + preStates.set(state._id.b, { + opId: compactedState.op_id, + checksum: { + bucket: state._id.b, + checksum: Number(compactedState.checksum), + count: compactedState.count } - results.set(request.bucket, merged ?? emptyChecksumForRequest(request)); - } + }); } - return results; + return preStates; + } + + protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + return this.computePartialChecksumsDirectByDefinition(this.normalizeBatch(batch)); } } @@ -419,9 +477,11 @@ class MongoChecksumsV3Impl extends AbstractMongoChecksums { */ export class MongoChecksums { private readonly impl: AbstractMongoChecksums; + private readonly v3Impl: MongoChecksumsV3Impl | null; + private readonly v1Impl: MongoChecksumsV1Impl | null; constructor(db: VersionedPowerSyncMongo, group_id: number, options: MongoChecksumOptions) { - this.impl = options.storageConfig.incrementalReprocessing + this.v3Impl = options.storageConfig.incrementalReprocessing ? new MongoChecksumsV3Impl( db, group_id, @@ -431,7 +491,9 @@ export class MongoChecksums { throw new ServiceAssertionError('BucketDefinitionMapping is required for v3 MongoDB checksum queries'); })() ) - : new MongoChecksumsV1Impl(db, group_id, options); + : null; + this.v1Impl = this.v3Impl == null ? new MongoChecksumsV1Impl(db, group_id, options) : null; + this.impl = this.v3Impl ?? this.v1Impl!; } async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise { @@ -445,9 +507,23 @@ export class MongoChecksums { async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise { return this.impl.computePartialChecksumsDirect(batch); } + + async computePartialChecksumsDirectV1(batch: FetchPartialBucketChecksumByBucket[]): Promise { + if (this.v1Impl == null) { + throw new ServiceAssertionError('V1 checksum routing is only available when incrementalReprocessing is disabled'); + } + return this.v1Impl.computePartialChecksumsDirectByBucket(batch); + } + + async computePartialChecksumsDirectV3(batch: FetchPartialBucketChecksumV3[]): Promise { + if (this.v3Impl == null) { + throw new ServiceAssertionError('V3 checksum routing is only available when incrementalReprocessing is enabled'); + } + return this.v3Impl.computePartialChecksumsDirectByDefinition(batch); + } } -function createV3BucketFilter(request: FetchPartialBucketChecksum) { +function createV3BucketFilter(request: Pick) { return { _id: { $gt: { @@ -462,18 +538,14 @@ function createV3BucketFilter(request: FetchPartialBucketChecksum) { }; } -function emptyChecksumForRequest(request: FetchPartialBucketChecksum): PartialOrFullChecksum { +function emptyChecksumForRequest( + request: Pick +): PartialOrFullChecksum { return request.start == null ? { bucket: request.bucket, count: 0, checksum: 0 } : { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }; } -function isBucketSourceLike( - source: FetchPartialBucketChecksum['source'] -): source is NonNullable { - return source != null && typeof source == 'object' && 'uniqueName' in source; -} - /** * Convert output of the $group stage into a checksum. */ diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index d29bc0ed9..8d465433d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -14,7 +14,8 @@ import { BucketDefinitionId } from './BucketDefinitionMapping.js'; import { BucketDataDocumentV1, BucketDataDocumentV3, - BucketStateDocument, + BucketStateDocumentV1, + BucketStateDocumentV3, LEGACY_BUCKET_DATA_DEFINITION_ID, TaggedBucketDataDocument, bucketDataDocumentToTagged, @@ -27,6 +28,7 @@ import { cacheKey } from './OperationBatch.js'; interface CurrentBucketState { /** Bucket name */ bucket: string; + definitionId: BucketDefinitionId; /** * Rows seen in the bucket, with the last op_id of each. @@ -94,9 +96,16 @@ const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; /** This default is primarily for tests. */ const DEFAULT_MEMORY_LIMIT_MB = 64; +interface DirtyBucket { + bucket: string; + definitionId: BucketDefinitionId | null; + estimatedCount: number; + dirtyRatio?: number; +} + export class MongoCompactor { private updates: mongo.AnyBulkWriteOperation[] = []; - private bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; + private bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; private activeBucketDataCollection: mongo.Collection | null = null; private activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; @@ -156,8 +165,8 @@ export class MongoCompactor { continue; } - for (let { bucket } of buckets) { - await this.compactSingleBucketRetried(bucket); + for (let { bucket, definitionId } of buckets) { + await this.compactSingleBucketRetried(bucket, definitionId); } } } @@ -167,11 +176,11 @@ export class MongoCompactor { * * This covers against occasional network or other database errors during a long compact job. */ - private async compactSingleBucketRetried(bucket: string) { + private async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { let retryCount = 0; while (true) { try { - await this.compactSingleBucket(bucket); + await this.compactSingleBucket(bucket, definitionId); break; } catch (e) { if (retryCount < 3 && isMongoServerError(e)) { @@ -185,9 +194,9 @@ export class MongoCompactor { } } - private async compactSingleBucket(bucket: string) { + private async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { const idLimitBytes = this.idLimitBytes; - const bucketCollection = await this.getBucketDataCollection(bucket); + const bucketCollection = await this.getBucketDataCollection(bucket, definitionId); if (bucketCollection == null) { return; } @@ -196,6 +205,7 @@ export class MongoCompactor { try { let currentState: CurrentBucketState = { bucket, + definitionId: bucketCollection.definitionId, seen: new Map(), trackingSize: 0, lastNotPut: null, @@ -371,12 +381,19 @@ export class MongoCompactor { } this.bucketStateUpdates.push({ updateOne: { - filter: { - _id: { - g: this.group_id, - b: state.bucket - } - }, + filter: this.db.storageConfig.incrementalReprocessing + ? { + _id: { + d: state.definitionId, + b: state.bucket + } + } + : { + _id: { + g: this.group_id, + b: state.bucket + } + }, update: { $set: { compacted_state: { @@ -418,9 +435,20 @@ export class MongoCompactor { } if (this.bucketStateUpdates.length > 0) { logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); - await this.db.bucket_state.bulkWrite(this.bucketStateUpdates, { - ordered: false - }); + if (this.db.storageConfig.incrementalReprocessing) { + await this.db + .bucketStateV3(this.group_id) + .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { + ordered: false + }); + } else { + await this.db.bucketStateV1.bulkWrite( + this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], + { + ordered: false + } + ); + } this.bucketStateUpdates = []; } } @@ -570,7 +598,7 @@ export class MongoCompactor { logger.info( `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` ); - await this.updateChecksumsBatch(checkBuckets.map((b) => b.bucket)); + await this.updateChecksumsBatch(checkBuckets); logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); count += checkBuckets.length; } @@ -588,7 +616,7 @@ export class MongoCompactor { private async *dirtyBucketBatches(options: { minBucketChanges: number; minChangeRatio: number; - }): AsyncGenerator<{ bucket: string; estimatedCount: number }[]> { + }): AsyncGenerator { // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only buckets with changes. // This works well if there are only a small number of buckets with changes. // However, if buckets are continuosly modified while we are compacting, we get the same buckets over and over again. @@ -598,16 +626,26 @@ export class MongoCompactor { if (options.minBucketChanges <= 0) { throw new ReplicationAssertionError('minBucketChanges must be >= 1'); } - let lastId = { g: this.group_id, b: new mongo.MinKey() as any }; - const maxId = { g: this.group_id, b: new mongo.MaxKey() as any }; + let lastId: mongo.Document = this.db.storageConfig.incrementalReprocessing + ? { d: new mongo.MinKey() as any, b: new mongo.MinKey() as any } + : { g: this.group_id, b: new mongo.MinKey() as any }; + const maxId: mongo.Document = this.db.storageConfig.incrementalReprocessing + ? { d: new mongo.MaxKey() as any, b: new mongo.MaxKey() as any } + : { g: this.group_id, b: new mongo.MaxKey() as any }; + const bucketState = this.db.storageConfig.incrementalReprocessing + ? this.db.bucketStateV3(this.group_id) + : this.db.bucketStateV1; while (true) { // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, we use an aggregation pipeline // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria, rather than limiting // on the output number. - const [result] = await this.db.bucket_state + const [result] = await bucketState .aggregate<{ - buckets: Pick[]; - cursor: Pick[]; + buckets: Pick< + BucketStateDocumentV1 | BucketStateDocumentV3, + '_id' | 'estimate_since_compact' | 'compacted_state' + >[]; + cursor: Pick[]; }>( [ { @@ -665,6 +703,7 @@ export class MongoCompactor { const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; return { bucket: b._id.b, + definitionId: 'd' in b._id ? b._id.d : null, estimatedCount: totalCount, dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) }; @@ -687,19 +726,22 @@ export class MongoCompactor { * * We currently don't get new data while doing populateChecksums, so we don't need to worry about buckets changing while processing. */ - private async dirtyBucketBatchForChecksums(options: { - minBucketChanges: number; - }): Promise<{ bucket: string; estimatedCount: number }[]> { + private async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { if (options.minBucketChanges <= 0) { throw new ReplicationAssertionError('minBucketChanges must be >= 1'); } - // We make use of an index on {_id.g: 1, 'estimate_since_compact.count': -1} - const dirtyBuckets = await this.db.bucket_state + const dirtyBuckets = await ( + this.db.storageConfig.incrementalReprocessing ? this.db.bucketStateV3(this.group_id) : this.db.bucketStateV1 + ) .find( - { - '_id.g': this.group_id, - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - }, + this.db.storageConfig.incrementalReprocessing + ? { + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + } + : { + '_id.g': this.group_id, + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + }, { projection: { _id: 1, @@ -717,20 +759,33 @@ export class MongoCompactor { return dirtyBuckets.map((bucket) => ({ bucket: bucket._id.b, + definitionId: 'd' in bucket._id ? bucket._id.d : null, estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) })); } - private async updateChecksumsBatch(buckets: string[]) { - const checksums = await this.storage.checksums.computePartialChecksumsDirect( - buckets.map((bucket) => { - return { - bucket, - source: {} as any, - end: this.maxOpId - }; - }) - ); + private async updateChecksumsBatch(buckets: Pick[]) { + const checksums = this.db.storageConfig.incrementalReprocessing + ? await this.storage.checksums.computePartialChecksumsDirectV3( + buckets.map(({ bucket, definitionId }) => { + if (definitionId == null) { + throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`); + } + return { + bucket, + definitionId, + end: this.maxOpId + }; + }) + ) + : await this.storage.checksums.computePartialChecksumsDirectV1( + buckets.map(({ bucket }) => { + return { + bucket, + end: this.maxOpId + }; + }) + ); for (let bucketChecksum of checksums.values()) { if (isPartialChecksum(bucketChecksum)) { @@ -740,12 +795,19 @@ export class MongoCompactor { this.bucketStateUpdates.push({ updateOne: { - filter: { - _id: { - g: this.group_id, - b: bucketChecksum.bucket - } - }, + filter: this.db.storageConfig.incrementalReprocessing + ? { + _id: { + d: buckets.find((bucket) => bucket.bucket === bucketChecksum.bucket)!.definitionId, + b: bucketChecksum.bucket + } + } + : { + _id: { + g: this.group_id, + b: bucketChecksum.bucket + } + }, update: { $set: { compacted_state: { @@ -782,8 +844,14 @@ export class MongoCompactor { }; } + /** + * FIXME: This is slow! + * + * Only used for compacting a single bucket. + */ private async getBucketDataCollection( - bucket: string + bucket: string, + definitionId: BucketDefinitionId | null = null ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { if (!this.db.storageConfig.incrementalReprocessing) { return { @@ -792,6 +860,13 @@ export class MongoCompactor { }; } + if (definitionId != null) { + return { + collection: this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, + definitionId + }; + } + for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { const existing = await collection.findOne( { '_id.b': bucket }, diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 5269abef6..9d8c7c2fd 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -112,6 +112,10 @@ export class MongoSyncBucketStorage return this.writeCheckpointAPI.writeCheckpointMode; } + get mapping() { + return this.sync_rules.mapping; + } + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { this.writeCheckpointAPI.setWriteCheckpointMode(mode); } @@ -992,17 +996,29 @@ export class MongoSyncBucketStorage await collection.drop(); } - await this.clearDeleteMany( - 'bucket state', - () => - this.db.bucket_state.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['b']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ), - signal - ); + if (this.db.storageConfig.incrementalReprocessing) { + await this.db + .bucketStateV3(this.group_id) + .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } else { + await this.clearDeleteMany( + 'bucket state', + () => + this.db.bucketStateV1.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['b']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } if (this.db.storageConfig.incrementalReprocessing) { await this.db @@ -1313,9 +1329,18 @@ export class MongoSyncBucketStorage private async getDataBucketChanges( options: GetCheckpointChangesOptions + ): Promise> { + if (this.db.storageConfig.incrementalReprocessing) { + return this.getDataBucketChangesV3(options); + } + return this.getDataBucketChangesV1(options); + } + + private async getDataBucketChangesV1( + options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; - const bucketStateUpdates = await this.db.bucket_state + const bucketStateUpdates = await this.db.bucketStateV1 .find( { // We have an index on (_id.g, last_op). @@ -1344,6 +1369,47 @@ export class MongoSyncBucketStorage }; } + private async getDataBucketChangesV3( + options: GetCheckpointChangesOptions + ): Promise> { + const limit = 1000; + const bucketStateUpdates = await this.db + .bucketStateV3(this.group_id) + .aggregate<{ _id: string; last_op: bigint }>( + [ + { + $match: { + last_op: { $gt: options.lastCheckpoint.checkpoint } + } + }, + { + $group: { + _id: '$_id.b', + last_op: { $max: '$last_op' } + } + }, + { + $sort: { + last_op: 1 + } + }, + { + $limit: limit + 1 + } + ], + { maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS } + ) + .toArray(); + + const buckets = bucketStateUpdates.map((doc) => doc._id); + const invalidateDataBuckets = buckets.length > limit; + + return { + invalidateDataBuckets, + updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets) + }; + } + private async getParameterBucketChanges( options: GetCheckpointChangesOptions ): Promise> { diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts index 49f942317..8a7ae38f0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts @@ -2,13 +2,18 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; +import { Logger, logger as defaultLogger, ReplicationAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { MongoIdSequence } from './MongoIdSequence.js'; import { VersionedPowerSyncMongo } from './db.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { BucketStateDocument, TaggedBucketParameterDocument, TaggedBucketDataDocument } from './models.js'; import { BucketDefinitionId } from './BucketDefinitionMapping.js'; +import { + BucketStateDocumentV1, + BucketStateDocumentV3, + TaggedBucketParameterDocument, + TaggedBucketDataDocument +} from './models.js'; import { mongoTableId } from '../../utils/util.js'; import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js'; @@ -122,14 +127,22 @@ export abstract class PersistedBatch { return this.bucketData.length; } - protected incrementBucket(bucket: string, op_id: InternalOpId, bytes: number) { - let existingState = this.bucketStates.get(bucket); + protected incrementBucket( + definitionId: BucketDefinitionId | null, + bucket: string, + op_id: InternalOpId, + bytes: number + ) { + const key = `${definitionId ?? ''}:${bucket}`; + let existingState = this.bucketStates.get(key); if (existingState) { existingState.lastOp = op_id; existingState.incrementCount += 1; existingState.incrementBytes += bytes; } else { - this.bucketStates.set(bucket, { + this.bucketStates.set(key, { + definitionId, + bucket, lastOp: op_id, incrementCount: 1, incrementBytes: bytes @@ -218,10 +231,17 @@ export abstract class PersistedBatch { if (this.bucketStates.size > 0) { flushedSomething = true; - await db.bucket_state.bulkWrite(this.getBucketStateUpdates(), { - session, - ordered: false - }); + if (db.storageConfig.incrementalReprocessing) { + await db.bucketStateV3(this.group_id).bulkWrite(this.getBucketStateUpdatesV3(), { + session, + ordered: false + }); + } else { + await db.bucketStateV1.bulkWrite(this.getBucketStateUpdatesV1(), { + session, + ordered: false + }); + } } if (flushedSomething) { @@ -279,14 +299,42 @@ export abstract class PersistedBatch { return stats; } - private getBucketStateUpdates(): mongo.AnyBulkWriteOperation[] { - return Array.from(this.bucketStates.entries()).map(([bucket, state]) => { + private getBucketStateUpdatesV1(): mongo.AnyBulkWriteOperation[] { + return Array.from(this.bucketStates.values()).map((state) => { return { updateOne: { filter: { _id: { g: this.group_id, - b: bucket + b: state.bucket + } + }, + update: { + $set: { + last_op: state.lastOp + }, + $inc: { + 'estimate_since_compact.count': state.incrementCount, + 'estimate_since_compact.bytes': state.incrementBytes + } + }, + upsert: true + } + } satisfies mongo.AnyBulkWriteOperation; + }); + } + + private getBucketStateUpdatesV3(): mongo.AnyBulkWriteOperation[] { + return Array.from(this.bucketStates.values()).map((state) => { + if (state.definitionId == null) { + throw new ReplicationAssertionError('Expected bucket definition id when incrementalReprocessing is enabled'); + } + return { + updateOne: { + filter: { + _id: { + d: state.definitionId, + b: state.bucket } }, update: { @@ -300,12 +348,14 @@ export abstract class PersistedBatch { }, upsert: true } - } satisfies mongo.AnyBulkWriteOperation; + } satisfies mongo.AnyBulkWriteOperation; }); } } interface BucketStateUpdate { + definitionId: BucketDefinitionId | null; + bucket: string; lastOp: InternalOpId; incrementCount: number; incrementBytes: number; diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts index 8d978c2d2..fcdb8e0c6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts @@ -69,7 +69,7 @@ export class PersistedBatchV1 extends PersistedBatch { checksum: BigInt(checksum), data: recordData }); - this.incrementBucket(evaluated.bucket, op_id, byteEstimate); + this.incrementBucket(null, evaluated.bucket, op_id, byteEstimate); } for (let bucket of remaining_buckets.values()) { @@ -87,7 +87,7 @@ export class PersistedBatchV1 extends PersistedBatch { checksum: dchecksum }); this.currentSize += 200; - this.incrementBucket(bucket.bucket, op_id, 200); + this.incrementBucket(null, bucket.bucket, op_id, 200); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts index 95aec78bb..7aba884ab 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts @@ -71,7 +71,7 @@ export class PersistedBatchV3 extends PersistedBatch { checksum: BigInt(checksum), data: recordData }); - this.incrementBucket(evaluated.bucket, op_id, byteEstimate); + this.incrementBucket(sourceDefinitionId, evaluated.bucket, op_id, byteEstimate); } for (let bucket of remaining_buckets.values()) { @@ -93,7 +93,7 @@ export class PersistedBatchV3 extends PersistedBatch { checksum: dchecksum }); this.currentSize += 200; - this.incrementBucket(bucket.bucket, op_id, 200); + this.incrementBucket(definitionId, bucket.bucket, op_id, 200); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index d75173088..d16330b61 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -11,6 +11,8 @@ import { BucketParameterDocument, BucketParameterDocumentV3, BucketStateDocument, + BucketStateDocumentV1, + BucketStateDocumentV3, CheckpointEventDocument, ClientConnectionDocument, CommonSourceTableDocument, @@ -44,7 +46,7 @@ export class PowerSyncMongo { readonly write_checkpoints: mongo.Collection; readonly instance: mongo.Collection; readonly locks: mongo.Collection; - readonly bucket_state: mongo.Collection; + readonly bucket_state: mongo.Collection; readonly checkpoint_events: mongo.Collection; readonly connection_report_events: mongo.Collection; @@ -95,6 +97,14 @@ export class PowerSyncMongo { .map((collection) => this.db.collection(collection.name)); } + bucketStateCollectionNameV3(replicationStreamId: number) { + return `bucket_state_${replicationStreamId}`; + } + + bucketStateV3(replicationStreamId: number): mongo.Collection { + return this.db.collection(this.bucketStateCollectionNameV3(replicationStreamId)); + } + bucketParameterCollectionNameV3(replicationStreamId: number, indexId: ParameterIndexId) { return `parameter_index_${replicationStreamId}_${indexId}`; } @@ -134,6 +144,10 @@ export class PowerSyncMongo { return this.collectionsByPrefix(`source_records_`); } + async listAllBucketStateCollectionsV3(): Promise[]> { + return this.collectionsByPrefix(`bucket_state_`); + } + sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) { return `source_records_${replicationStreamId}_${sourceTableId.toHexString()}`; } @@ -173,6 +187,9 @@ export class PowerSyncMongo { for (const collection of await this.listAllParameterIndexCollectionsV3()) { await collection.drop(); } + for (const collection of await this.listAllBucketStateCollectionsV3()) { + await collection.drop(); + } await this.op_id_sequence.deleteMany({}); await this.sync_rules.deleteMany({}); for (const collection of await this.listSourceTableCollections()) { @@ -302,6 +319,15 @@ export class VersionedPowerSyncMongo { return this.#upstream.current_data; } + get bucketStateV1() { + if (this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'bucket_state collection should not be used when incrementalReprocessing is enabled' + ); + } + return this.#upstream.bucket_state; + } + sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId) { if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( @@ -347,6 +373,15 @@ export class VersionedPowerSyncMongo { } } + bucketStateV3(replicationStreamId: number) { + if (!this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError( + 'v3 bucket_state collection should not be used when incrementalReprocessing is disabled' + ); + } + return this.#upstream.bucketStateV3(replicationStreamId); + } + sourceTablesV3(replicationStreamId: number) { if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( @@ -359,6 +394,7 @@ export class VersionedPowerSyncMongo { async initializeStreamStorage(replicationStreamId: number) { if (this.storageConfig.incrementalReprocessing) { const sourceTables = this.sourceTablesV3(replicationStreamId); + const bucketState = this.bucketStateV3(replicationStreamId); await sourceTables.createIndex( { connection_id: 1, @@ -379,6 +415,18 @@ export class VersionedPowerSyncMongo { name: 'latest_pending_delete' } ); + await bucketState.createIndex( + { + last_op: 1 + }, + { name: 'bucket_updates', unique: true } + ); + await bucketState.createIndex( + { + 'estimate_since_compact.count': -1 + }, + { name: 'dirty_count' } + ); } } @@ -478,10 +526,6 @@ export class VersionedPowerSyncMongo { return this.#upstream.locks; } - get bucket_state() { - return this.#upstream.bucket_state; - } - get checkpoint_events() { return this.#upstream.checkpoint_events; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index e3b507fd7..07aa04e9e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -231,13 +231,12 @@ export interface SourceTableDocumentSnapshotStatus { * Note: There is currently no migration to populate this collection from existing data - it is only * populated by new updates. */ -export interface BucketStateDocument { +interface BucketStateDocumentBase { _id: { - g: number; b: string; }; /** - * Important: There is an unique index on {'_id.g': 1, last_op: 1}. + * Important: There is an unique index on last_op per logical stream. * That means the last_op must match an actual op in the bucket, and not the commit checkpoint. */ last_op: bigint; @@ -258,6 +257,20 @@ export interface BucketStateDocument { }; } +export interface BucketStateDocumentV1 extends BucketStateDocumentBase { + _id: BucketStateDocumentBase['_id'] & { + g: number; + }; +} + +export interface BucketStateDocumentV3 extends BucketStateDocumentBase { + _id: BucketStateDocumentBase['_id'] & { + d: BucketDefinitionId; + }; +} + +export type BucketStateDocument = BucketStateDocumentV1; + export interface IdSequenceDocument { _id: string; op_id: bigint; diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index e277f6413..a4b1f3d4f 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -64,9 +64,14 @@ bucket_definitions: test('full compact', async () => { const { bucketStorage, checkpoint, factory, syncRules } = await setup(); + const storageDb = (bucketStorage as any).db; // Simulate bucket_state from old version not being available - await factory.db.bucket_state.deleteMany({}); + if (storageDb.storageConfig.incrementalReprocessing) { + await storageDb.bucketStateV3(bucketStorage.group_id).deleteMany({}); + } else { + await factory.db.bucket_state.deleteMany({}); + } await bucketStorage.compact({ clearBatchLimit: 200, @@ -108,6 +113,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const storageDb = (bucketStorage as any).db; await populate(bucketStorage, 2); const { checkpoint } = await bucketStorage.getCheckpoint(); @@ -158,27 +164,52 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const storageDb = (bucketStorage as any).db; // This simulates bucket_state created using bigint bytes. // This typically happens when buckets get very large (> 2GiB). We don't want to create that much // data in the tests, so we directly insert the bucket_state here. - await factory.db.bucket_state.insertOne({ - _id: { - g: bucketStorage.group_id, - b: 'global[]' - }, - last_op: 5n, - compacted_state: { - op_id: 3n, - count: 3, - checksum: 0n, - bytes: 7n - }, - estimate_since_compact: { - count: 2, - bytes: 5n - } - }); + await ( + storageDb.storageConfig.incrementalReprocessing + ? storageDb.bucketStateV3(bucketStorage.group_id) + : factory.db.bucket_state + ).insertOne( + storageDb.storageConfig.incrementalReprocessing + ? { + _id: { + d: '1', + b: 'global[]' + }, + last_op: 5n, + compacted_state: { + op_id: 3n, + count: 3, + checksum: 0n, + bytes: 7n + }, + estimate_since_compact: { + count: 2, + bytes: 5n + } + } + : { + _id: { + g: bucketStorage.group_id, + b: 'global[]' + }, + last_op: 5n, + compacted_state: { + op_id: 3n, + count: 3, + checksum: 0n, + bytes: 7n + }, + estimate_since_compact: { + count: 2, + bytes: 5n + } + } + ); // This test uses a couple of internal APIs of the compactor - there is no simple way // to test this using the current public APIs. @@ -205,6 +236,7 @@ bucket_definitions: expect(checksumBuckets).toEqual([ { bucket: 'global[]', + definitionId: storageDb.storageConfig.incrementalReprocessing ? '1' : null, estimatedCount: 5 } ]); From 24218af0386eedc43fc33ea92030eb41296d3967 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 14:52:50 +0200 Subject: [PATCH 51/93] Split compactor implementations. --- .../storage/implementation/MongoChecksums.ts | 37 +- .../storage/implementation/MongoCompactor.ts | 869 ++++++++++-------- .../test/src/storage_compacting.test.ts | 83 +- 3 files changed, 530 insertions(+), 459 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index bd889c857..46f4348da 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -481,19 +481,23 @@ export class MongoChecksums { private readonly v1Impl: MongoChecksumsV1Impl | null; constructor(db: VersionedPowerSyncMongo, group_id: number, options: MongoChecksumOptions) { - this.v3Impl = options.storageConfig.incrementalReprocessing - ? new MongoChecksumsV3Impl( - db, - group_id, - options, - options.mapping ?? - (() => { - throw new ServiceAssertionError('BucketDefinitionMapping is required for v3 MongoDB checksum queries'); - })() - ) - : null; - this.v1Impl = this.v3Impl == null ? new MongoChecksumsV1Impl(db, group_id, options) : null; - this.impl = this.v3Impl ?? this.v1Impl!; + if (options.storageConfig.incrementalReprocessing) { + this.v3Impl = new MongoChecksumsV3Impl( + db, + group_id, + options, + options.mapping ?? + (() => { + throw new ServiceAssertionError('BucketDefinitionMapping is required for v3 MongoDB checksum queries'); + })() + ); + this.v1Impl = null; + this.impl = this.v3Impl; + } else { + this.v3Impl = null; + this.v1Impl = new MongoChecksumsV1Impl(db, group_id, options); + this.impl = this.v1Impl; + } } async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise { @@ -541,9 +545,10 @@ function createV3BucketFilter(request: Pick ): PartialOrFullChecksum { - return request.start == null - ? { bucket: request.bucket, count: 0, checksum: 0 } - : { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }; + if (request.start == null) { + return { bucket: request.bucket, count: 0, checksum: 0 }; + } + return { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }; } /** diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 8d465433d..33d0c95d5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -29,7 +29,6 @@ interface CurrentBucketState { /** Bucket name */ bucket: string; definitionId: BucketDefinitionId; - /** * Rows seen in the bucket, with the last op_id of each. */ @@ -38,27 +37,22 @@ interface CurrentBucketState { * Estimated memory usage of the seen Map. */ trackingSize: number; - /** * Last (lowest) seen op_id that is not a PUT. */ lastNotPut: InternalOpId | null; - /** * Number of REMOVE/MOVE operations seen since lastNotPut. */ opsSincePut: number; - /** - * Incrementally-updated checksum, up to maxOpId + * Incrementally-updated checksum, up to maxOpId. */ checksum: number; - /** - * op count for the checksum + * Op count for the checksum. */ opCount: number; - /** * Byte size of ops covered by the checksum. */ @@ -81,9 +75,18 @@ type BucketDataClearProjection = { target_op?: bigint | null; }; -/** - * Additional options, primarily for testing. - */ +type BucketStateProjection = { + _id: { b: string }; + estimate_since_compact?: { + count: number; + bytes: number | bigint; + }; + compacted_state?: { + count: number; + bytes: number | bigint | null; + }; +}; + export interface MongoCompactOptions extends storage.CompactOptions {} const DEFAULT_CLEAR_BATCH_LIMIT = 5000; @@ -92,7 +95,6 @@ const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; const DEFAULT_MIN_BUCKET_CHANGES = 10; const DEFAULT_MIN_CHANGE_RATIO = 0.1; const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; - /** This default is primarily for tests. */ const DEFAULT_MEMORY_LIMIT_MB = 64; @@ -103,26 +105,26 @@ interface DirtyBucket { dirtyRatio?: number; } -export class MongoCompactor { - private updates: mongo.AnyBulkWriteOperation[] = []; - private bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; - private activeBucketDataCollection: mongo.Collection | null = null; - private activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; - - private idLimitBytes: number; - private moveBatchLimit: number; - private moveBatchQueryLimit: number; - private clearBatchLimit: number; - private minBucketChanges: number; - private minChangeRatio: number; - private maxOpId: bigint; - private buckets: string[] | undefined; - private signal?: AbortSignal; - private group_id: number; +abstract class BaseMongoCompactor { + protected updates: mongo.AnyBulkWriteOperation[] = []; + protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; + protected activeBucketDataCollection: mongo.Collection | null = null; + protected activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; + + protected readonly idLimitBytes: number; + protected readonly moveBatchLimit: number; + protected readonly moveBatchQueryLimit: number; + protected readonly clearBatchLimit: number; + protected readonly minBucketChanges: number; + protected readonly minChangeRatio: number; + protected readonly maxOpId: bigint; + protected readonly buckets: string[] | undefined; + protected readonly signal?: AbortSignal; + protected readonly group_id: number; constructor( - private storage: MongoSyncBucketStorage, - private db: VersionedPowerSyncMongo, + protected readonly storage: MongoSyncBucketStorage, + protected readonly db: VersionedPowerSyncMongo, options: MongoCompactOptions ) { this.group_id = storage.group_id; @@ -144,9 +146,8 @@ export class MongoCompactor { */ async compact() { if (this.buckets) { - for (let bucket of this.buckets) { - // We can make this more efficient later on by iterating - // through the buckets in a single query. + for (const bucket of this.buckets) { + // We can make this more efficient later on by iterating through the buckets in a single query. // That makes batching more tricky, so we leave for later. await this.compactSingleBucketRetried(bucket); } @@ -155,8 +156,161 @@ export class MongoCompactor { } } - private async compactDirtyBuckets() { - for await (let buckets of this.dirtyBucketBatches({ + /** + * Subset of compact, only populating checksums where relevant. + */ + async populateChecksums(options: { minBucketChanges: number }): Promise { + let count = 0; + while (true) { + this.signal?.throwIfAborted(); + const buckets = await this.dirtyBucketBatchForChecksums(options); + if (buckets.length == 0) { + break; + } + this.signal?.throwIfAborted(); + + const start = Date.now(); + // Filter batch by estimated bucket size, to reduce possibility of timeouts. + const checkBuckets: typeof buckets = []; + let totalCountEstimate = 0; + for (const bucket of buckets) { + checkBuckets.push(bucket); + totalCountEstimate += bucket.estimatedCount; + if (totalCountEstimate > 50_000) { + break; + } + } + logger.info( + `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` + ); + await this.updateChecksumsBatch(checkBuckets); + logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); + count += checkBuckets.length; + } + return { buckets: count }; + } + + protected async *dirtyBucketBatchesForCollection( + collection: mongo.Collection, + lastId: mongo.Document, + maxId: mongo.Document, + options: { + minBucketChanges: number; + minChangeRatio: number; + }, + getDefinitionId: (state: TBucketState) => BucketDefinitionId | null + ): AsyncGenerator { + while (true) { + // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline + // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria. + const [result] = await collection + .aggregate<{ + buckets: TBucketState[]; + cursor: Pick[]; + }>( + [ + { + $match: { + _id: { $gt: lastId, $lt: maxId } + } + }, + { + $sort: { _id: 1 } + }, + { + // Scan a fixed number of docs each query so sparse matches don't block progress. + $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE + }, + { + $facet: { + buckets: [ + { + $match: { + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + } + }, + { + $project: { + _id: 1, + estimate_since_compact: 1, + compacted_state: 1 + } + } + ], + // This is used for the next query. + cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] + } + } + ], + { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } + ) + .toArray(); + + const cursor = result?.cursor?.[0]; + if (cursor == null) { + break; + } + lastId = cursor._id as mongo.Document; + + const mapped = (result?.buckets ?? []).map((bucketState) => { + // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios. + // BigInt precision is not needed here since this is only an estimate. + const updatedCount = bucketState.estimate_since_compact?.count ?? 0; + const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; + const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); + const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; + const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; + const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; + return { + bucket: bucketState._id.b, + definitionId: getDefinitionId(bucketState), + estimatedCount: totalCount, + dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) + }; + }); + + yield mapped.filter( + (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio + ); + } + } + + protected async dirtyBucketBatchForChecksumsForCollection( + collection: mongo.Collection, + filter: mongo.Filter, + getDefinitionId: (state: mongo.WithId) => BucketDefinitionId | null + ): Promise { + const dirtyBuckets = await collection + .find(filter, { + projection: { + _id: 1, + estimate_since_compact: 1, + compacted_state: 1 + }, + sort: { + 'estimate_since_compact.count': -1 + }, + limit: 200, + maxTimeMS: MONGO_OPERATION_TIMEOUT_MS + }) + .toArray(); + + return dirtyBuckets.map((bucket) => ({ + bucket: bucket._id.b, + definitionId: getDefinitionId(bucket), + estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) + })); + } + + public abstract dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator; + + public abstract dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise; + + protected async compactDirtyBuckets() { + for await (const buckets of this.dirtyBucketBatches({ minBucketChanges: this.minBucketChanges, minChangeRatio: this.minChangeRatio })) { @@ -165,7 +319,7 @@ export class MongoCompactor { continue; } - for (let { bucket, definitionId } of buckets) { + for (const { bucket, definitionId } of buckets) { await this.compactSingleBucketRetried(bucket, definitionId); } } @@ -176,7 +330,7 @@ export class MongoCompactor { * * This covers against occasional network or other database errors during a long compact job. */ - private async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { + protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { let retryCount = 0; while (true) { try { @@ -194,7 +348,7 @@ export class MongoCompactor { } } - private async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { + protected async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { const idLimitBytes = this.idLimitBytes; const bucketCollection = await this.getBucketDataCollection(bucket, definitionId); if (bucketCollection == null) { @@ -203,29 +357,27 @@ export class MongoCompactor { this.activeBucketDataCollection = bucketCollection.collection; this.activeBucketDefinitionId = bucketCollection.definitionId; try { - let currentState: CurrentBucketState = { + const currentState: CurrentBucketState = { bucket, definitionId: bucketCollection.definitionId, seen: new Map(), trackingSize: 0, lastNotPut: null, opsSincePut: 0, - checksum: 0, opCount: 0, opBytes: 0 }; - // Constant lower bound + // Constant lower bound. const lowerBound = this.bucketDataKey(bucket, new mongo.MinKey() as any); - - // Upper bound is adjusted for each batch + // Upper bound is adjusted for each batch. let upperBound = this.bucketDataKey(bucket, new mongo.MaxKey() as any); while (true) { this.signal?.throwIfAborted(); - // Query one batch at a time, to avoid cursor timeouts + // Query one batch at a time, to avoid cursor timeouts. const pipeline = [ { $match: { @@ -233,8 +385,7 @@ export class MongoCompactor { $gte: lowerBound, $lt: upperBound }, - // Workaround for bug with clustered collections (storage v3), where the $lt operator - // may include the upperBound. + // Workaround for a clustered collection bug where the $lt operator may include upperBound. // https://jira.mongodb.org/browse/SERVER-121822 '_id.o': { $lt: upperBound.o } } @@ -269,14 +420,14 @@ export class MongoCompactor { const batch = rawBatch.map((document) => this.tagBucketDataDocument(document, bucketCollection.definitionId)); if (batch.length == 0) { - // We've reached the end + // We've reached the end. break; } - // Reuse the exact collection _id value from Mongo for the next bound + // Reuse the exact collection _id value from Mongo for the next bound. upperBound = rawBatch[rawBatch.length - 1]._id; - for (let doc of batch) { + for (const doc of batch) { if (doc._id.o > this.maxOpId) { continue; } @@ -291,7 +442,7 @@ export class MongoCompactor { const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; const targetOp = currentState.seen.get(key); if (targetOp) { - // Will convert to MOVE, so don't count as PUT + // Will convert to MOVE, so don't count as PUT. isPersistentPut = false; this.updates.push({ @@ -313,20 +464,16 @@ export class MongoCompactor { } }); - currentState.opBytes += 200 - Number(doc.size); // TODO: better estimate for this - } else { - if (currentState.trackingSize >= idLimitBytes) { - // Reached memory limit. - // Keep the highest seen values in this case. - } else { - // flatstr reduces the memory usage by flattening the string - currentState.seen.set(utils.flatstr(key), doc._id.o); - // length + 16 for the string - // 24 for the bigint - // 50 for map overhead - // 50 for additional overhead - currentState.trackingSize += key.length + 140; - } + // TODO: better estimate for this. + currentState.opBytes += 200 - Number(doc.size); + } else if (currentState.trackingSize < idLimitBytes) { + // flatstr reduces the memory usage by flattening the string. + currentState.seen.set(utils.flatstr(key), doc._id.o); + // length + 16 for the string + // 24 for the bigint + // 50 for map overhead + // 50 for additional overhead + currentState.trackingSize += key.length + 140; } } @@ -348,21 +495,20 @@ export class MongoCompactor { logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); } - // Free memory before clearing bucket + // Free memory before clearing the bucket. currentState.seen.clear(); if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { logger.info( `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` ); - // Need flush() before clear() + // Need flush() before clear(). await this.flush(); await this.clearBucket(currentState); } - // Do this _after_ clearBucket so that we have accurate counts. + // Do this after clearBucket so we have accurate counts. this.updateBucketChecksums(currentState); - - // Need another flush after updateBucketChecksums() + // Need another flush after updateBucketChecksums(). await this.flush(); } finally { this.activeBucketDataCollection = null; @@ -370,10 +516,7 @@ export class MongoCompactor { } } - /** - * Call when done with a bucket. - */ - private updateBucketChecksums(state: CurrentBucketState) { + protected updateBucketChecksums(state: CurrentBucketState) { if (state.opCount < 0) { throw new ServiceAssertionError( `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` @@ -381,19 +524,7 @@ export class MongoCompactor { } this.bucketStateUpdates.push({ updateOne: { - filter: this.db.storageConfig.incrementalReprocessing - ? { - _id: { - d: state.definitionId, - b: state.bucket - } - } - : { - _id: { - g: this.group_id, - b: state.bucket - } - }, + filter: this.bucketStateFilter(state.bucket, state.definitionId), update: { $set: { compacted_state: { @@ -403,9 +534,8 @@ export class MongoCompactor { bytes: state.opBytes }, estimate_since_compact: { - // Note: There could have been a whole bunch of new operations added to the bucket _while_ compacting, - // which we don't currently cater for. - // We could potentially query for that, but that could add overhead. + // There could have been a whole bunch of new operations added to the bucket while compacting, + // which we don't currently cater for. We could potentially query for that, but that adds overhead. count: 0, bytes: 0 } @@ -418,37 +548,22 @@ export class MongoCompactor { }); } - private async flush() { + protected async flush() { if (this.updates.length > 0) { logger.info(`Compacting ${this.updates.length} ops`); if (this.activeBucketDataCollection == null) { throw new ServiceAssertionError('No bucket_data collection selected for compaction'); } await this.activeBucketDataCollection.bulkWrite(this.updates, { - // Order is not important. - // Since checksums are not affected, these operations can happen in any order, - // and it's fine if the operations are partially applied. - // Each individual operation is atomic. + // Order is not important. Since checksums are not affected, these operations can happen in any order, + // and it's fine if the operations are partially applied. Each individual operation is atomic. ordered: false }); this.updates = []; } if (this.bucketStateUpdates.length > 0) { logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); - if (this.db.storageConfig.incrementalReprocessing) { - await this.db - .bucketStateV3(this.group_id) - .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { - ordered: false - }); - } else { - await this.db.bucketStateV1.bulkWrite( - this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], - { - ordered: false - } - ); - } + await this.flushBucketStateUpdates(); this.bucketStateUpdates = []; } } @@ -456,11 +571,9 @@ export class MongoCompactor { /** * Perform a CLEAR compact for a bucket. * - * - * @param bucket bucket name - * @param op op_id of the last non-PUT operation, which will be converted to CLEAR. + * @param currentState tracks the last non-PUT op, which will be converted to CLEAR. */ - private async clearBucket(currentState: CurrentBucketState) { + protected async clearBucket(currentState: CurrentBucketState) { const bucket = currentState.bucket; const clearOp = currentState.lastNotPut!; const bucketCollection = this.activeBucketDataCollection; @@ -502,7 +615,7 @@ export class MongoCompactor { let targetOp: bigint | null = null; let gotAnOp = false; let numberOfOpsToClear = 0; - for await (let rawOp of query.stream()) { + for await (const rawOp of query.stream()) { const op = this.tagClearBucketDataDocument( rawOp as unknown as BucketDataClearProjection, this.activeBucketDefinitionId @@ -515,10 +628,8 @@ export class MongoCompactor { if (op.op != 'CLEAR') { gotAnOp = true; } - if (op.target_op != null) { - if (targetOp == null || op.target_op > targetOp) { - targetOp = op.target_op; - } + if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) { + targetOp = op.target_op; } } else { throw new ReplicationAssertionError( @@ -561,7 +672,7 @@ export class MongoCompactor { readConcern: { level: 'snapshot' } } ); - // Update _outside_ the transaction, since the transaction can be retried multiple times. + // Update outside the transaction, since the transaction can be retried multiple times. currentState.opCount += opCountDiff; } } finally { @@ -569,245 +680,22 @@ export class MongoCompactor { } } - /** - * Subset of compact, only populating checksums where relevant. - */ - async populateChecksums(options: { minBucketChanges: number }): Promise { - let count = 0; - while (true) { - this.signal?.throwIfAborted(); - const buckets = await this.dirtyBucketBatchForChecksums(options); - if (buckets.length == 0) { - // All done - break; - } - this.signal?.throwIfAborted(); - - const start = Date.now(); - - // Filter batch by estimated bucket size, to reduce possibility of timeouts - let checkBuckets: typeof buckets = []; - let totalCountEstimate = 0; - for (let bucket of buckets) { - checkBuckets.push(bucket); - totalCountEstimate += bucket.estimatedCount; - if (totalCountEstimate > 50_000) { - break; - } - } - logger.info( - `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` - ); - await this.updateChecksumsBatch(checkBuckets); - logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); - count += checkBuckets.length; - } - return { buckets: count }; - } - - /** - * Return batches of dirty buckets. - * - * Can be used to iterate through all buckets. - * - * minBucketChanges: minimum number of changes for a bucket to be included in the results. - * minChangeRatio: minimum ratio of changes to total ops for a bucket to be included in the results, number between 0 and 1. - */ - private async *dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator { - // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only buckets with changes. - // This works well if there are only a small number of buckets with changes. - // However, if buckets are continuosly modified while we are compacting, we get the same buckets over and over again. - // This has caused the compact process to re-read the same collection around 5x times in total, which is very inefficient. - // To solve this, we now just iterate through all buckets, and filter out the ones with low changes. + protected async updateChecksumsBatch(buckets: Pick[]) { + const checksums = await this.computeChecksumsForBuckets(buckets); + const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); - } - let lastId: mongo.Document = this.db.storageConfig.incrementalReprocessing - ? { d: new mongo.MinKey() as any, b: new mongo.MinKey() as any } - : { g: this.group_id, b: new mongo.MinKey() as any }; - const maxId: mongo.Document = this.db.storageConfig.incrementalReprocessing - ? { d: new mongo.MaxKey() as any, b: new mongo.MaxKey() as any } - : { g: this.group_id, b: new mongo.MaxKey() as any }; - const bucketState = this.db.storageConfig.incrementalReprocessing - ? this.db.bucketStateV3(this.group_id) - : this.db.bucketStateV1; - while (true) { - // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, we use an aggregation pipeline - // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria, rather than limiting - // on the output number. - const [result] = await bucketState - .aggregate<{ - buckets: Pick< - BucketStateDocumentV1 | BucketStateDocumentV3, - '_id' | 'estimate_since_compact' | 'compacted_state' - >[]; - cursor: Pick[]; - }>( - [ - { - $match: { - _id: { $gt: lastId, $lt: maxId } - } - }, - { - $sort: { _id: 1 } - }, - { - // Scan a fixed number of docs each query so sparse matches don't block progress. - $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE - }, - { - $facet: { - // This is the results for the batch - buckets: [ - { - $match: { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - } - }, - { - $project: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - } - } - ], - // This is used for the next query. - cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] - } - } - ], - { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } - ) - .toArray(); - - const cursor = result?.cursor?.[0]; - if (cursor == null) { - break; - } - lastId = cursor._id; - - const mapped = (result?.buckets ?? []).map((b) => { - // The numbers, specifically the bytes, could be a bigint. We convert to Number to allow calculating the ratios. - // BigInt precision is not needed here since it's just an estimate. - const updatedCount = b.estimate_since_compact?.count ?? 0; - const totalCount = (b.compacted_state?.count ?? 0) + updatedCount; - const updatedBytes = Number(b.estimate_since_compact?.bytes ?? 0); - const totalBytes = Number(b.compacted_state?.bytes ?? 0) + updatedBytes; - const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; - const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; - return { - bucket: b._id.b, - definitionId: 'd' in b._id ? b._id.d : null, - estimatedCount: totalCount, - dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) - }; - }); - const filtered = mapped.filter( - (b) => b.estimatedCount >= options.minBucketChanges && b.dirtyRatio >= options.minChangeRatio - ); - yield filtered; - } - } - - /** - * Returns a batch of dirty buckets - buckets with most changes first. - * - * This cannot be used to iterate on its own - the client is expected to process these buckets and - * set estimate_since_compact.count: 0 when done, before fetching the next batch. - * - * Unlike dirtyBucketBatches, used for compacting, this is specifically designed to be resuamble after a restart, - * since it is used as the last step for initial replication. - * - * We currently don't get new data while doing populateChecksums, so we don't need to worry about buckets changing while processing. - */ - private async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); - } - const dirtyBuckets = await ( - this.db.storageConfig.incrementalReprocessing ? this.db.bucketStateV3(this.group_id) : this.db.bucketStateV1 - ) - .find( - this.db.storageConfig.incrementalReprocessing - ? { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - } - : { - '_id.g': this.group_id, - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - }, - { - projection: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - }, - sort: { - 'estimate_since_compact.count': -1 - }, - limit: 200, - maxTimeMS: MONGO_OPERATION_TIMEOUT_MS - } - ) - .toArray(); - - return dirtyBuckets.map((bucket) => ({ - bucket: bucket._id.b, - definitionId: 'd' in bucket._id ? bucket._id.d : null, - estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) - })); - } - - private async updateChecksumsBatch(buckets: Pick[]) { - const checksums = this.db.storageConfig.incrementalReprocessing - ? await this.storage.checksums.computePartialChecksumsDirectV3( - buckets.map(({ bucket, definitionId }) => { - if (definitionId == null) { - throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`); - } - return { - bucket, - definitionId, - end: this.maxOpId - }; - }) - ) - : await this.storage.checksums.computePartialChecksumsDirectV1( - buckets.map(({ bucket }) => { - return { - bucket, - end: this.maxOpId - }; - }) - ); - - for (let bucketChecksum of checksums.values()) { + for (const bucketChecksum of checksums.values()) { if (isPartialChecksum(bucketChecksum)) { - // Should never happen since we don't specify `start` + // Should never happen since we don't specify `start`. throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); } this.bucketStateUpdates.push({ updateOne: { - filter: this.db.storageConfig.incrementalReprocessing - ? { - _id: { - d: buckets.find((bucket) => bucket.bucket === bucketChecksum.bucket)!.definitionId, - b: bucketChecksum.bucket - } - } - : { - _id: { - g: this.group_id, - b: bucketChecksum.bucket - } - }, + filter: this.bucketStateFilter( + bucketChecksum.bucket, + definitionIdByBucket.get(bucketChecksum.bucket) ?? null + ), update: { $set: { compacted_state: { @@ -822,8 +710,8 @@ export class MongoCompactor { } } }, - // We don't create new ones here - it gets tricky to get the last_op right with the unique index on: - // bucket_updates: {'id.g': 1, 'last_op': 1} + // We don't create new ones here - it gets tricky to get the last_op right with the unique index on + // bucket_updates. upsert: false } }); @@ -832,11 +720,120 @@ export class MongoCompactor { await this.flush(); } - private bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey) { - if (this.db.storageConfig.incrementalReprocessing) { - return { b: bucket, o: opId as any }; + protected tagBucketDataDocument( + document: BucketDataCollectionDocument & { size: number | bigint }, + definitionId: BucketDefinitionId + ): CompactBucketDataDocument { + const tagged = bucketDataDocumentToTagged(document, definitionId); + return { + ...tagged, + size: document.size + }; + } + + protected tagClearBucketDataDocument( + document: BucketDataClearProjection, + definitionId: BucketDefinitionId + ): CompactClearBucketDataDocument { + return { + def: definitionId, + _id: { + b: document._id.b, + o: document._id.o + }, + op: document.op, + checksum: document.checksum, + target_op: document.target_op + }; + } + + protected formatBucketDataKey(key: mongo.Document) { + const bucket = (key.b ?? key._id?.b) as string | undefined; + const op = (key.o ?? key._id?.o) as bigint | undefined; + return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; + } + + protected abstract flushBucketStateUpdates(): Promise; + protected abstract computeChecksumsForBuckets( + buckets: Pick[] + ): Promise; + protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; + protected abstract bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document; + protected abstract getBucketDataCollection( + bucket: string, + definitionId: BucketDefinitionId | null + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null>; + protected abstract collectionBucketDataDocument( + document: TaggedBucketDataDocument + ): BucketDataDocumentV1 | BucketDataDocumentV3; +} + +class MongoCompactorV1 extends BaseMongoCompactor { + public async *dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only scan buckets with changes. + // That works well if there are only a small number of dirty buckets, but it causes repeated rescans while data is + // still changing. We now iterate through all V1 bucket_state rows for the group and filter after projecting. + yield* this.dirtyBucketBatchesForCollection( + this.db.bucketStateV1, + { g: this.group_id, b: new mongo.MinKey() as any }, + { g: this.group_id, b: new mongo.MaxKey() as any }, + options, + () => null + ); + } + + public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); } + // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets + // estimate_since_compact as it progresses. + return this.dirtyBucketBatchForChecksumsForCollection( + this.db.bucketStateV1, + { + '_id.g': this.group_id, + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + }, + () => null + ); + } + protected async flushBucketStateUpdates(): Promise { + await this.db.bucketStateV1.bulkWrite( + this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], + { + ordered: false + } + ); + } + + protected async computeChecksumsForBuckets( + buckets: Pick[] + ): Promise { + return this.storage.checksums.computePartialChecksumsDirectV1( + buckets.map(({ bucket }) => ({ + bucket, + end: this.maxOpId + })) + ); + } + + protected bucketStateFilter(bucket: string, _definitionId: BucketDefinitionId | null): mongo.Document { + return { + _id: { + g: this.group_id, + b: bucket + } + }; + } + + protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document { return { g: this.group_id, b: bucket, @@ -844,22 +841,97 @@ export class MongoCompactor { }; } - /** - * FIXME: This is slow! - * - * Only used for compacting a single bucket. - */ - private async getBucketDataCollection( - bucket: string, - definitionId: BucketDefinitionId | null = null + protected async getBucketDataCollection( + _bucket: string, + _definitionId: BucketDefinitionId | null ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { - if (!this.db.storageConfig.incrementalReprocessing) { - return { - collection: this.db.v1_bucket_data as unknown as mongo.Collection, - definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID - }; + return { + collection: this.db.v1_bucket_data as unknown as mongo.Collection, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID + }; + } + + protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentV1 { + return taggedBucketDataDocumentToV1(this.group_id, document); + } +} + +class MongoCompactorV3 extends BaseMongoCompactor { + public async *dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + // Same scan strategy as V1, but with the V3 bucket_state key shape. + yield* this.dirtyBucketBatchesForCollection( + this.db.bucketStateV3(this.group_id), + { d: new mongo.MinKey() as any, b: new mongo.MinKey() as any }, + { d: new mongo.MaxKey() as any, b: new mongo.MaxKey() as any }, + options, + (bucketState) => (bucketState as BucketStateDocumentV3)._id.d + ); + } + + public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + return this.dirtyBucketBatchForChecksumsForCollection( + this.db.bucketStateV3(this.group_id), + { + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + }, + (bucketState) => (bucketState as BucketStateDocumentV3)._id.d + ); + } + + protected async flushBucketStateUpdates(): Promise { + await this.db + .bucketStateV3(this.group_id) + .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { + ordered: false + }); + } + + protected async computeChecksumsForBuckets( + buckets: Pick[] + ): Promise { + return this.storage.checksums.computePartialChecksumsDirectV3( + buckets.map(({ bucket, definitionId }) => { + if (definitionId == null) { + throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`); + } + return { + bucket, + definitionId, + end: this.maxOpId + }; + }) + ); + } + + protected bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document { + if (definitionId == null) { + throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`); } + return { + _id: { + d: definitionId, + b: bucket + } + }; + } + + protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document { + return { b: bucket, o: opId as any }; + } + protected async getBucketDataCollection( + bucket: string, + definitionId: BucketDefinitionId | null + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { if (definitionId != null) { return { collection: this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, @@ -867,16 +939,17 @@ export class MongoCompactor { }; } + // FIXME: This is slow. It is only used when compacting a single bucket without a known definition id. for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { const existing = await collection.findOne( { '_id.b': bucket }, { projection: { _id: 1 }, maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } ); if (existing != null) { - const definitionId = collection.collectionName.replace(`bucket_data_${this.group_id}_`, ''); + const resolvedDefinitionId = collection.collectionName.replace(`bucket_data_${this.group_id}_`, ''); return { collection: collection as unknown as mongo.Collection, - definitionId + definitionId: resolvedDefinitionId }; } } @@ -884,43 +957,35 @@ export class MongoCompactor { return null; } - private formatBucketDataKey(key: mongo.Document) { - const bucket = (key.b ?? key._id?.b) as string | undefined; - const op = (key.o ?? key._id?.o) as bigint | undefined; - return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; + protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentV3 { + return taggedBucketDataDocumentToV3(document); } +} - private tagBucketDataDocument( - document: BucketDataCollectionDocument & { size: number | bigint }, - definitionId: BucketDefinitionId - ): CompactBucketDataDocument { - const tagged = bucketDataDocumentToTagged(document, definitionId); - return { - ...tagged, - size: document.size - }; +export class MongoCompactor { + private readonly impl: BaseMongoCompactor; + + constructor(storage: MongoSyncBucketStorage, db: VersionedPowerSyncMongo, options: MongoCompactOptions) { + if (db.storageConfig.incrementalReprocessing) { + this.impl = new MongoCompactorV3(storage, db, options); + } else { + this.impl = new MongoCompactorV1(storage, db, options); + } } - private tagClearBucketDataDocument( - document: BucketDataClearProjection, - definitionId: BucketDefinitionId - ): CompactClearBucketDataDocument { - return { - def: definitionId, - _id: { - b: document._id.b, - o: document._id.o - }, - op: document.op, - checksum: document.checksum, - target_op: document.target_op - }; + async compact() { + return this.impl.compact(); } - private collectionBucketDataDocument(document: TaggedBucketDataDocument) { - if (this.db.storageConfig.incrementalReprocessing) { - return taggedBucketDataDocumentToV3(document); - } - return taggedBucketDataDocumentToV1(this.group_id, document); + async populateChecksums(options: { minBucketChanges: number }): Promise { + return this.impl.populateChecksums(options); + } + + dirtyBucketBatches(options: { minBucketChanges: number; minChangeRatio: number }): AsyncGenerator { + return this.impl.dirtyBucketBatches(options); + } + + dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { + return this.impl.dirtyBucketBatchForChecksums(options); } } diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index a4b1f3d4f..fed79a826 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -169,47 +169,48 @@ bucket_definitions: // This simulates bucket_state created using bigint bytes. // This typically happens when buckets get very large (> 2GiB). We don't want to create that much // data in the tests, so we directly insert the bucket_state here. - await ( - storageDb.storageConfig.incrementalReprocessing - ? storageDb.bucketStateV3(bucketStorage.group_id) - : factory.db.bucket_state - ).insertOne( - storageDb.storageConfig.incrementalReprocessing - ? { - _id: { - d: '1', - b: 'global[]' - }, - last_op: 5n, - compacted_state: { - op_id: 3n, - count: 3, - checksum: 0n, - bytes: 7n - }, - estimate_since_compact: { - count: 2, - bytes: 5n - } - } - : { - _id: { - g: bucketStorage.group_id, - b: 'global[]' - }, - last_op: 5n, - compacted_state: { - op_id: 3n, - count: 3, - checksum: 0n, - bytes: 7n - }, - estimate_since_compact: { - count: 2, - bytes: 5n - } - } - ); + let bucketStateCollection; + let bucketStateDocument; + if (storageDb.storageConfig.incrementalReprocessing) { + bucketStateCollection = storageDb.bucketStateV3(bucketStorage.group_id); + bucketStateDocument = { + _id: { + d: '1', + b: 'global[]' + }, + last_op: 5n, + compacted_state: { + op_id: 3n, + count: 3, + checksum: 0n, + bytes: 7n + }, + estimate_since_compact: { + count: 2, + bytes: 5n + } + }; + } else { + bucketStateCollection = factory.db.bucket_state; + bucketStateDocument = { + _id: { + g: bucketStorage.group_id, + b: 'global[]' + }, + last_op: 5n, + compacted_state: { + op_id: 3n, + count: 3, + checksum: 0n, + bytes: 7n + }, + estimate_since_compact: { + count: 2, + bytes: 5n + } + }; + } + await bucketStateCollection.insertOne(bucketStateDocument); // This test uses a couple of internal APIs of the compactor - there is no simple way // to test this using the current public APIs. From 4410aa9b3617ec09d83b5434a872346fd061b1d0 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 15:09:15 +0200 Subject: [PATCH 52/93] Avoid listing collections unless we drop them. --- .../src/storage/implementation/MongoSyncBucketStorage.ts | 7 +++++-- .../src/storage/implementation/db.ts | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 9d8c7c2fd..0c067371c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -1455,7 +1455,11 @@ export class MongoSyncBucketStorage options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; - const collections = await this.db.listParameterIndexCollectionsV3(this.group_id); + const indexIds = this.mapping.allParameterIndexIds(); + const collections = indexIds.map((indexId) => ({ + indexId, + collection: this.db.parameterIndexV3(this.group_id, indexId) + })); if (collections.length == 0) { return { invalidateParameterBuckets: false, @@ -1465,7 +1469,6 @@ export class MongoSyncBucketStorage const checkpointFilter = { _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint } }; - const collectionPrefix = `parameter_index_${this.group_id}_`; const pipelineForCollection = (indexId: string) => [ { $match: checkpointFilter diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index d16330b61..ee62b0e9f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -452,7 +452,7 @@ export class VersionedPowerSyncMongo { return this.#upstream.bucketDataV3(groupId, definitionId); } - listBucketDataCollectionsV3(groupId?: number) { + listBucketDataCollectionsV3(groupId: number) { if (!this.storageConfig.incrementalReprocessing) { throw new ServiceAssertionError( 'v3 bucket_data collections should not be used when incrementalReprocessing is disabled' From a32ff380295a1ab27659472958ad67ee6f943c27 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 15:20:01 +0200 Subject: [PATCH 53/93] Split implementation versions, phase 1. --- .../implementation/MongoBucketBatchShared.ts | 2 +- .../storage/implementation/MongoCompactor.ts | 992 +----------------- .../implementation/MongoSyncBucketStorage.ts | 6 +- .../{ => common}/MongoBucketBatch.ts | 16 +- .../implementation/common/MongoCompactor.ts | 36 + .../common/MongoCompactorBase.ts | 765 ++++++++++++++ .../{ => common}/PersistedBatch.ts | 12 +- .../{ => common}/SourceRecordStore.ts | 2 +- .../{ => v1}/MongoBucketBatchV1.ts | 6 +- .../implementation/v1/MongoCompactorV1.ts | 100 ++ .../{ => v1}/PersistedBatchV1.ts | 8 +- .../{ => v1}/SourceRecordStoreV1.ts | 10 +- .../{ => v3}/MongoBucketBatchV3.ts | 6 +- .../implementation/v3/MongoCompactorV3.ts | 117 +++ .../{ => v3}/MongoParameterLookupV3.ts | 2 +- .../{ => v3}/PersistedBatchV3.ts | 10 +- .../{ => v3}/SourceRecordStoreV3.ts | 12 +- .../src/storage/storage-index.ts | 2 +- .../test/src/storage_sync.test.ts | 2 +- 19 files changed, 1067 insertions(+), 1039 deletions(-) rename modules/module-mongodb-storage/src/storage/implementation/{ => common}/MongoBucketBatch.ts (98%) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts rename modules/module-mongodb-storage/src/storage/implementation/{ => common}/PersistedBatch.ts (96%) rename modules/module-mongodb-storage/src/storage/implementation/{ => common}/SourceRecordStore.ts (94%) rename modules/module-mongodb-storage/src/storage/implementation/{ => v1}/MongoBucketBatchV1.ts (74%) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts rename modules/module-mongodb-storage/src/storage/implementation/{ => v1}/PersistedBatchV1.ts (96%) rename modules/module-mongodb-storage/src/storage/implementation/{ => v1}/SourceRecordStoreV1.ts (94%) rename modules/module-mongodb-storage/src/storage/implementation/{ => v3}/MongoBucketBatchV3.ts (75%) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts rename modules/module-mongodb-storage/src/storage/implementation/{ => v3}/MongoParameterLookupV3.ts (88%) rename modules/module-mongodb-storage/src/storage/implementation/{ => v3}/PersistedBatchV3.ts (97%) rename modules/module-mongodb-storage/src/storage/implementation/{ => v3}/SourceRecordStoreV3.ts (95%) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts index 64c0e8427..54ee38691 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchShared.ts @@ -1,5 +1,5 @@ import * as bson from 'bson'; -import { SourceRecordBucketState } from './SourceRecordStore.js'; +import { SourceRecordBucketState } from './common/SourceRecordStore.js'; export const MAX_ROW_SIZE = 15 * 1024 * 1024; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 33d0c95d5..46ea63083 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -1,991 +1 @@ -import { isMongoServerError, mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb'; -import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; -import { - addChecksums, - InternalOpId, - isPartialChecksum, - PopulateChecksumCacheResults, - storage, - utils -} from '@powersync/service-core'; - -import { VersionedPowerSyncMongo } from './db.js'; -import { BucketDefinitionId } from './BucketDefinitionMapping.js'; -import { - BucketDataDocumentV1, - BucketDataDocumentV3, - BucketStateDocumentV1, - BucketStateDocumentV3, - LEGACY_BUCKET_DATA_DEFINITION_ID, - TaggedBucketDataDocument, - bucketDataDocumentToTagged, - taggedBucketDataDocumentToV1, - taggedBucketDataDocumentToV3 -} from './models.js'; -import { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; -import { cacheKey } from './OperationBatch.js'; - -interface CurrentBucketState { - /** Bucket name */ - bucket: string; - definitionId: BucketDefinitionId; - /** - * Rows seen in the bucket, with the last op_id of each. - */ - seen: Map; - /** - * Estimated memory usage of the seen Map. - */ - trackingSize: number; - /** - * Last (lowest) seen op_id that is not a PUT. - */ - lastNotPut: InternalOpId | null; - /** - * Number of REMOVE/MOVE operations seen since lastNotPut. - */ - opsSincePut: number; - /** - * Incrementally-updated checksum, up to maxOpId. - */ - checksum: number; - /** - * Op count for the checksum. - */ - opCount: number; - /** - * Byte size of ops covered by the checksum. - */ - opBytes: number; -} - -type CompactBucketDataDocument = Pick< - TaggedBucketDataDocument, - '_id' | 'def' | 'op' | 'table' | 'row_id' | 'source_table' | 'source_key' | 'checksum' | 'target_op' -> & { - size: number | bigint; -}; - -type CompactClearBucketDataDocument = Pick; -type BucketDataCollectionDocument = BucketDataDocumentV1 | BucketDataDocumentV3; -type BucketDataClearProjection = { - _id: BucketDataCollectionDocument['_id']; - op: CompactClearBucketDataDocument['op']; - checksum: bigint; - target_op?: bigint | null; -}; - -type BucketStateProjection = { - _id: { b: string }; - estimate_since_compact?: { - count: number; - bytes: number | bigint; - }; - compacted_state?: { - count: number; - bytes: number | bigint | null; - }; -}; - -export interface MongoCompactOptions extends storage.CompactOptions {} - -const DEFAULT_CLEAR_BATCH_LIMIT = 5000; -const DEFAULT_MOVE_BATCH_LIMIT = 2000; -const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; -const DEFAULT_MIN_BUCKET_CHANGES = 10; -const DEFAULT_MIN_CHANGE_RATIO = 0.1; -const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; -/** This default is primarily for tests. */ -const DEFAULT_MEMORY_LIMIT_MB = 64; - -interface DirtyBucket { - bucket: string; - definitionId: BucketDefinitionId | null; - estimatedCount: number; - dirtyRatio?: number; -} - -abstract class BaseMongoCompactor { - protected updates: mongo.AnyBulkWriteOperation[] = []; - protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; - protected activeBucketDataCollection: mongo.Collection | null = null; - protected activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; - - protected readonly idLimitBytes: number; - protected readonly moveBatchLimit: number; - protected readonly moveBatchQueryLimit: number; - protected readonly clearBatchLimit: number; - protected readonly minBucketChanges: number; - protected readonly minChangeRatio: number; - protected readonly maxOpId: bigint; - protected readonly buckets: string[] | undefined; - protected readonly signal?: AbortSignal; - protected readonly group_id: number; - - constructor( - protected readonly storage: MongoSyncBucketStorage, - protected readonly db: VersionedPowerSyncMongo, - options: MongoCompactOptions - ) { - this.group_id = storage.group_id; - this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; - this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; - this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; - this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; - this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; - this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; - this.maxOpId = options.maxOpId ?? 0n; - this.buckets = options.compactBuckets; - this.signal = options.signal; - } - - /** - * Compact buckets by converting operations into MOVE and/or CLEAR operations. - * - * See /docs/compacting-operations.md for details. - */ - async compact() { - if (this.buckets) { - for (const bucket of this.buckets) { - // We can make this more efficient later on by iterating through the buckets in a single query. - // That makes batching more tricky, so we leave for later. - await this.compactSingleBucketRetried(bucket); - } - } else { - await this.compactDirtyBuckets(); - } - } - - /** - * Subset of compact, only populating checksums where relevant. - */ - async populateChecksums(options: { minBucketChanges: number }): Promise { - let count = 0; - while (true) { - this.signal?.throwIfAborted(); - const buckets = await this.dirtyBucketBatchForChecksums(options); - if (buckets.length == 0) { - break; - } - this.signal?.throwIfAborted(); - - const start = Date.now(); - // Filter batch by estimated bucket size, to reduce possibility of timeouts. - const checkBuckets: typeof buckets = []; - let totalCountEstimate = 0; - for (const bucket of buckets) { - checkBuckets.push(bucket); - totalCountEstimate += bucket.estimatedCount; - if (totalCountEstimate > 50_000) { - break; - } - } - logger.info( - `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` - ); - await this.updateChecksumsBatch(checkBuckets); - logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); - count += checkBuckets.length; - } - return { buckets: count }; - } - - protected async *dirtyBucketBatchesForCollection( - collection: mongo.Collection, - lastId: mongo.Document, - maxId: mongo.Document, - options: { - minBucketChanges: number; - minChangeRatio: number; - }, - getDefinitionId: (state: TBucketState) => BucketDefinitionId | null - ): AsyncGenerator { - while (true) { - // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline - // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria. - const [result] = await collection - .aggregate<{ - buckets: TBucketState[]; - cursor: Pick[]; - }>( - [ - { - $match: { - _id: { $gt: lastId, $lt: maxId } - } - }, - { - $sort: { _id: 1 } - }, - { - // Scan a fixed number of docs each query so sparse matches don't block progress. - $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE - }, - { - $facet: { - buckets: [ - { - $match: { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - } - }, - { - $project: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - } - } - ], - // This is used for the next query. - cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] - } - } - ], - { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } - ) - .toArray(); - - const cursor = result?.cursor?.[0]; - if (cursor == null) { - break; - } - lastId = cursor._id as mongo.Document; - - const mapped = (result?.buckets ?? []).map((bucketState) => { - // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios. - // BigInt precision is not needed here since this is only an estimate. - const updatedCount = bucketState.estimate_since_compact?.count ?? 0; - const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; - const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); - const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; - const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; - const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; - return { - bucket: bucketState._id.b, - definitionId: getDefinitionId(bucketState), - estimatedCount: totalCount, - dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) - }; - }); - - yield mapped.filter( - (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio - ); - } - } - - protected async dirtyBucketBatchForChecksumsForCollection( - collection: mongo.Collection, - filter: mongo.Filter, - getDefinitionId: (state: mongo.WithId) => BucketDefinitionId | null - ): Promise { - const dirtyBuckets = await collection - .find(filter, { - projection: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - }, - sort: { - 'estimate_since_compact.count': -1 - }, - limit: 200, - maxTimeMS: MONGO_OPERATION_TIMEOUT_MS - }) - .toArray(); - - return dirtyBuckets.map((bucket) => ({ - bucket: bucket._id.b, - definitionId: getDefinitionId(bucket), - estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) - })); - } - - public abstract dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator; - - public abstract dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise; - - protected async compactDirtyBuckets() { - for await (const buckets of this.dirtyBucketBatches({ - minBucketChanges: this.minBucketChanges, - minChangeRatio: this.minChangeRatio - })) { - this.signal?.throwIfAborted(); - if (buckets.length == 0) { - continue; - } - - for (const { bucket, definitionId } of buckets) { - await this.compactSingleBucketRetried(bucket, definitionId); - } - } - } - - /** - * Compaction for a single bucket, with retries on failure. - * - * This covers against occasional network or other database errors during a long compact job. - */ - protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { - let retryCount = 0; - while (true) { - try { - await this.compactSingleBucket(bucket, definitionId); - break; - } catch (e) { - if (retryCount < 3 && isMongoServerError(e)) { - logger.warn(`Error compacting bucket ${bucket}, retrying...`, e); - retryCount++; - await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount)); - } else { - throw e; - } - } - } - } - - protected async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { - const idLimitBytes = this.idLimitBytes; - const bucketCollection = await this.getBucketDataCollection(bucket, definitionId); - if (bucketCollection == null) { - return; - } - this.activeBucketDataCollection = bucketCollection.collection; - this.activeBucketDefinitionId = bucketCollection.definitionId; - try { - const currentState: CurrentBucketState = { - bucket, - definitionId: bucketCollection.definitionId, - seen: new Map(), - trackingSize: 0, - lastNotPut: null, - opsSincePut: 0, - checksum: 0, - opCount: 0, - opBytes: 0 - }; - - // Constant lower bound. - const lowerBound = this.bucketDataKey(bucket, new mongo.MinKey() as any); - // Upper bound is adjusted for each batch. - let upperBound = this.bucketDataKey(bucket, new mongo.MaxKey() as any); - - while (true) { - this.signal?.throwIfAborted(); - - // Query one batch at a time, to avoid cursor timeouts. - const pipeline = [ - { - $match: { - _id: { - $gte: lowerBound, - $lt: upperBound - }, - // Workaround for a clustered collection bug where the $lt operator may include upperBound. - // https://jira.mongodb.org/browse/SERVER-121822 - '_id.o': { $lt: upperBound.o } - } - }, - { $sort: { _id: -1 } }, - { $limit: this.moveBatchQueryLimit }, - { - $project: { - _id: 1, - op: 1, - table: 1, - row_id: 1, - source_table: 1, - source_key: 1, - checksum: 1, - size: { $bsonSize: '$$ROOT' } - } - } - ]; - - const cursor = bucketCollection.collection.aggregate( - pipeline, - { - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: this.moveBatchQueryLimit + 1 - } - ); - // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. - // Instead, we load up to the limit. - const rawBatch = await cursor.toArray(); - const batch = rawBatch.map((document) => this.tagBucketDataDocument(document, bucketCollection.definitionId)); - - if (batch.length == 0) { - // We've reached the end. - break; - } - - // Reuse the exact collection _id value from Mongo for the next bound. - upperBound = rawBatch[rawBatch.length - 1]._id; - - for (const doc of batch) { - if (doc._id.o > this.maxOpId) { - continue; - } - - currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum)); - currentState.opCount += 1; - - let isPersistentPut = doc.op == 'PUT'; - - currentState.opBytes += Number(doc.size); - if (doc.op == 'REMOVE' || doc.op == 'PUT') { - const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; - const targetOp = currentState.seen.get(key); - if (targetOp) { - // Will convert to MOVE, so don't count as PUT. - isPersistentPut = false; - - this.updates.push({ - updateOne: { - filter: { _id: this.bucketDataKey(doc._id.b, doc._id.o) }, - update: { - $set: { - op: 'MOVE', - target_op: targetOp - }, - $unset: { - source_table: 1, - source_key: 1, - table: 1, - row_id: 1, - data: 1 - } - } - } - }); - - // TODO: better estimate for this. - currentState.opBytes += 200 - Number(doc.size); - } else if (currentState.trackingSize < idLimitBytes) { - // flatstr reduces the memory usage by flattening the string. - currentState.seen.set(utils.flatstr(key), doc._id.o); - // length + 16 for the string - // 24 for the bigint - // 50 for map overhead - // 50 for additional overhead - currentState.trackingSize += key.length + 140; - } - } - - if (isPersistentPut) { - currentState.lastNotPut = null; - currentState.opsSincePut = 0; - } else if (doc.op != 'CLEAR') { - if (currentState.lastNotPut == null) { - currentState.lastNotPut = doc._id.o; - } - currentState.opsSincePut += 1; - } - - if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { - await this.flush(); - } - } - - logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); - } - - // Free memory before clearing the bucket. - currentState.seen.clear(); - if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { - logger.info( - `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` - ); - // Need flush() before clear(). - await this.flush(); - await this.clearBucket(currentState); - } - - // Do this after clearBucket so we have accurate counts. - this.updateBucketChecksums(currentState); - // Need another flush after updateBucketChecksums(). - await this.flush(); - } finally { - this.activeBucketDataCollection = null; - this.activeBucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; - } - } - - protected updateBucketChecksums(state: CurrentBucketState) { - if (state.opCount < 0) { - throw new ServiceAssertionError( - `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` - ); - } - this.bucketStateUpdates.push({ - updateOne: { - filter: this.bucketStateFilter(state.bucket, state.definitionId), - update: { - $set: { - compacted_state: { - op_id: this.maxOpId, - count: state.opCount, - checksum: BigInt(state.checksum), - bytes: state.opBytes - }, - estimate_since_compact: { - // There could have been a whole bunch of new operations added to the bucket while compacting, - // which we don't currently cater for. We could potentially query for that, but that adds overhead. - count: 0, - bytes: 0 - } - } - }, - // We generally expect this to have been created before. - // We don't create new ones here, to avoid issues with the unique index on bucket_updates. - upsert: false - } - }); - } - - protected async flush() { - if (this.updates.length > 0) { - logger.info(`Compacting ${this.updates.length} ops`); - if (this.activeBucketDataCollection == null) { - throw new ServiceAssertionError('No bucket_data collection selected for compaction'); - } - await this.activeBucketDataCollection.bulkWrite(this.updates, { - // Order is not important. Since checksums are not affected, these operations can happen in any order, - // and it's fine if the operations are partially applied. Each individual operation is atomic. - ordered: false - }); - this.updates = []; - } - if (this.bucketStateUpdates.length > 0) { - logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); - await this.flushBucketStateUpdates(); - this.bucketStateUpdates = []; - } - } - - /** - * Perform a CLEAR compact for a bucket. - * - * @param currentState tracks the last non-PUT op, which will be converted to CLEAR. - */ - protected async clearBucket(currentState: CurrentBucketState) { - const bucket = currentState.bucket; - const clearOp = currentState.lastNotPut!; - const bucketCollection = this.activeBucketDataCollection; - if (bucketCollection == null) { - throw new ServiceAssertionError('No bucket_data collection selected for compaction'); - } - - const opFilter = { - _id: { - $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), - $lte: this.bucketDataKey(bucket, clearOp) - } - }; - - const session = this.db.client.startSession(); - try { - let done = false; - while (!done) { - this.signal?.throwIfAborted(); - let opCountDiff = 0; - // Do the CLEAR operation in batches, with each batch a separate transaction. - // The state after each batch is fully consistent. - // We need a transaction per batch to make sure checksums stay consistent. - await session.withTransaction( - async () => { - const query = bucketCollection.find(opFilter as any, { - session, - sort: { _id: 1 }, - projection: { - _id: 1, - op: 1, - checksum: 1, - target_op: 1 - }, - limit: this.clearBatchLimit - }); - let checksum = 0; - let lastOp: CompactClearBucketDataDocument | null = null; - let targetOp: bigint | null = null; - let gotAnOp = false; - let numberOfOpsToClear = 0; - for await (const rawOp of query.stream()) { - const op = this.tagClearBucketDataDocument( - rawOp as unknown as BucketDataClearProjection, - this.activeBucketDefinitionId - ); - - if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') { - checksum = utils.addChecksums(checksum, Number(op.checksum)); - lastOp = op; - numberOfOpsToClear += 1; - if (op.op != 'CLEAR') { - gotAnOp = true; - } - if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) { - targetOp = op.target_op; - } - } else { - throw new ReplicationAssertionError( - `Unexpected ${op.op} operation at ${this.formatBucketDataKey(op._id as unknown as mongo.Document)}` - ); - } - } - if (!gotAnOp) { - done = true; - return; - } - - logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?._id.o}`); - await bucketCollection.deleteMany( - { - _id: { - $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), - $lte: this.bucketDataKey(lastOp!._id.b, lastOp!._id.o) - } - } as any, - { session } as any - ); - - await bucketCollection.insertOne( - this.collectionBucketDataDocument({ - def: this.activeBucketDefinitionId, - _id: lastOp!._id, - op: 'CLEAR', - checksum: BigInt(checksum), - data: null, - target_op: targetOp - }) as unknown as mongo.OptionalId, - { session } as any - ); - - opCountDiff = -numberOfOpsToClear + 1; - }, - { - writeConcern: { w: 'majority' }, - readConcern: { level: 'snapshot' } - } - ); - // Update outside the transaction, since the transaction can be retried multiple times. - currentState.opCount += opCountDiff; - } - } finally { - await session.endSession(); - } - } - - protected async updateChecksumsBatch(buckets: Pick[]) { - const checksums = await this.computeChecksumsForBuckets(buckets); - const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); - - for (const bucketChecksum of checksums.values()) { - if (isPartialChecksum(bucketChecksum)) { - // Should never happen since we don't specify `start`. - throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); - } - - this.bucketStateUpdates.push({ - updateOne: { - filter: this.bucketStateFilter( - bucketChecksum.bucket, - definitionIdByBucket.get(bucketChecksum.bucket) ?? null - ), - update: { - $set: { - compacted_state: { - op_id: this.maxOpId, - count: bucketChecksum.count, - checksum: BigInt(bucketChecksum.checksum), - bytes: null - }, - estimate_since_compact: { - count: 0, - bytes: 0 - } - } - }, - // We don't create new ones here - it gets tricky to get the last_op right with the unique index on - // bucket_updates. - upsert: false - } - }); - } - - await this.flush(); - } - - protected tagBucketDataDocument( - document: BucketDataCollectionDocument & { size: number | bigint }, - definitionId: BucketDefinitionId - ): CompactBucketDataDocument { - const tagged = bucketDataDocumentToTagged(document, definitionId); - return { - ...tagged, - size: document.size - }; - } - - protected tagClearBucketDataDocument( - document: BucketDataClearProjection, - definitionId: BucketDefinitionId - ): CompactClearBucketDataDocument { - return { - def: definitionId, - _id: { - b: document._id.b, - o: document._id.o - }, - op: document.op, - checksum: document.checksum, - target_op: document.target_op - }; - } - - protected formatBucketDataKey(key: mongo.Document) { - const bucket = (key.b ?? key._id?.b) as string | undefined; - const op = (key.o ?? key._id?.o) as bigint | undefined; - return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; - } - - protected abstract flushBucketStateUpdates(): Promise; - protected abstract computeChecksumsForBuckets( - buckets: Pick[] - ): Promise; - protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; - protected abstract bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document; - protected abstract getBucketDataCollection( - bucket: string, - definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null>; - protected abstract collectionBucketDataDocument( - document: TaggedBucketDataDocument - ): BucketDataDocumentV1 | BucketDataDocumentV3; -} - -class MongoCompactorV1 extends BaseMongoCompactor { - public async *dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); - } - // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only scan buckets with changes. - // That works well if there are only a small number of dirty buckets, but it causes repeated rescans while data is - // still changing. We now iterate through all V1 bucket_state rows for the group and filter after projecting. - yield* this.dirtyBucketBatchesForCollection( - this.db.bucketStateV1, - { g: this.group_id, b: new mongo.MinKey() as any }, - { g: this.group_id, b: new mongo.MaxKey() as any }, - options, - () => null - ); - } - - public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); - } - // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets - // estimate_since_compact as it progresses. - return this.dirtyBucketBatchForChecksumsForCollection( - this.db.bucketStateV1, - { - '_id.g': this.group_id, - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - }, - () => null - ); - } - - protected async flushBucketStateUpdates(): Promise { - await this.db.bucketStateV1.bulkWrite( - this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], - { - ordered: false - } - ); - } - - protected async computeChecksumsForBuckets( - buckets: Pick[] - ): Promise { - return this.storage.checksums.computePartialChecksumsDirectV1( - buckets.map(({ bucket }) => ({ - bucket, - end: this.maxOpId - })) - ); - } - - protected bucketStateFilter(bucket: string, _definitionId: BucketDefinitionId | null): mongo.Document { - return { - _id: { - g: this.group_id, - b: bucket - } - }; - } - - protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document { - return { - g: this.group_id, - b: bucket, - o: opId as any - }; - } - - protected async getBucketDataCollection( - _bucket: string, - _definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { - return { - collection: this.db.v1_bucket_data as unknown as mongo.Collection, - definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID - }; - } - - protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentV1 { - return taggedBucketDataDocumentToV1(this.group_id, document); - } -} - -class MongoCompactorV3 extends BaseMongoCompactor { - public async *dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); - } - // Same scan strategy as V1, but with the V3 bucket_state key shape. - yield* this.dirtyBucketBatchesForCollection( - this.db.bucketStateV3(this.group_id), - { d: new mongo.MinKey() as any, b: new mongo.MinKey() as any }, - { d: new mongo.MaxKey() as any, b: new mongo.MaxKey() as any }, - options, - (bucketState) => (bucketState as BucketStateDocumentV3)._id.d - ); - } - - public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { - if (options.minBucketChanges <= 0) { - throw new ReplicationAssertionError('minBucketChanges must be >= 1'); - } - return this.dirtyBucketBatchForChecksumsForCollection( - this.db.bucketStateV3(this.group_id), - { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - }, - (bucketState) => (bucketState as BucketStateDocumentV3)._id.d - ); - } - - protected async flushBucketStateUpdates(): Promise { - await this.db - .bucketStateV3(this.group_id) - .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { - ordered: false - }); - } - - protected async computeChecksumsForBuckets( - buckets: Pick[] - ): Promise { - return this.storage.checksums.computePartialChecksumsDirectV3( - buckets.map(({ bucket, definitionId }) => { - if (definitionId == null) { - throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`); - } - return { - bucket, - definitionId, - end: this.maxOpId - }; - }) - ); - } - - protected bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document { - if (definitionId == null) { - throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`); - } - return { - _id: { - d: definitionId, - b: bucket - } - }; - } - - protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document { - return { b: bucket, o: opId as any }; - } - - protected async getBucketDataCollection( - bucket: string, - definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { - if (definitionId != null) { - return { - collection: this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, - definitionId - }; - } - - // FIXME: This is slow. It is only used when compacting a single bucket without a known definition id. - for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { - const existing = await collection.findOne( - { '_id.b': bucket }, - { projection: { _id: 1 }, maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } - ); - if (existing != null) { - const resolvedDefinitionId = collection.collectionName.replace(`bucket_data_${this.group_id}_`, ''); - return { - collection: collection as unknown as mongo.Collection, - definitionId: resolvedDefinitionId - }; - } - } - - return null; - } - - protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentV3 { - return taggedBucketDataDocumentToV3(document); - } -} - -export class MongoCompactor { - private readonly impl: BaseMongoCompactor; - - constructor(storage: MongoSyncBucketStorage, db: VersionedPowerSyncMongo, options: MongoCompactOptions) { - if (db.storageConfig.incrementalReprocessing) { - this.impl = new MongoCompactorV3(storage, db, options); - } else { - this.impl = new MongoCompactorV1(storage, db, options); - } - } - - async compact() { - return this.impl.compact(); - } - - async populateChecksums(options: { minBucketChanges: number }): Promise { - return this.impl.populateChecksums(options); - } - - dirtyBucketBatches(options: { minBucketChanges: number; minChangeRatio: number }): AsyncGenerator { - return this.impl.dirtyBucketBatches(options); - } - - dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { - return this.impl.dirtyBucketBatchForChecksums(options); - } -} +export * from './common/MongoCompactor.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 0c067371c..589386352 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -50,14 +50,14 @@ import { StorageConfig, bucketDataDocumentToTagged } from './models.js'; -import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; -import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; +import { MongoBucketBatchV1 } from './v1/MongoBucketBatchV1.js'; +import { MongoBucketBatchV3 } from './v3/MongoBucketBatchV3.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; -import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; +import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './v3/MongoParameterLookupV3.js'; export interface MongoSyncBucketStorageOptions { checksumOptions?: Omit; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts similarity index 98% rename from modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts rename to modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts index 44a4ffafc..25f0c06c9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts @@ -25,16 +25,16 @@ import { utils } from '@powersync/service-core'; import * as timers from 'node:timers/promises'; -import { mongoTableId } from '../../utils/util.js'; -import { VersionedPowerSyncMongo } from './db.js'; -import { SyncRuleDocument } from './models.js'; +import { mongoTableId } from '../../../utils/util.js'; +import { VersionedPowerSyncMongo } from '../db.js'; +import { SyncRuleDocument } from '../models.js'; import { LoadedSourceRecord, SourceRecordStore } from './SourceRecordStore.js'; -import { MongoIdSequence } from './MongoIdSequence.js'; -import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; -import { OperationBatch, RecordOperation } from './OperationBatch.js'; +import { MongoIdSequence } from '../MongoIdSequence.js'; +import { batchCreateCustomWriteCheckpoints } from '../MongoWriteCheckpointAPI.js'; +import { OperationBatch, RecordOperation } from '../OperationBatch.js'; import { PersistedBatch } from './PersistedBatch.js'; -import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; +import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; // Currently, we can only have a single flush() at a time, since it locks the op_id sequence. // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts new file mode 100644 index 000000000..95a7daa82 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts @@ -0,0 +1,36 @@ +import { PopulateChecksumCacheResults } from '@powersync/service-core'; +import { VersionedPowerSyncMongo } from '../db.js'; +import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; +import { MongoCompactorV1 } from '../v1/MongoCompactorV1.js'; +import { MongoCompactorV3 } from '../v3/MongoCompactorV3.js'; +import { BaseMongoCompactor, DirtyBucket, MongoCompactOptions } from './MongoCompactorBase.js'; + +export { DirtyBucket, MongoCompactOptions } from './MongoCompactorBase.js'; + +export class MongoCompactor { + private readonly impl: BaseMongoCompactor; + + constructor(storage: MongoSyncBucketStorage, db: VersionedPowerSyncMongo, options: MongoCompactOptions) { + if (db.storageConfig.incrementalReprocessing) { + this.impl = new MongoCompactorV3(storage, db, options); + } else { + this.impl = new MongoCompactorV1(storage, db, options); + } + } + + async compact() { + return this.impl.compact(); + } + + async populateChecksums(options: { minBucketChanges: number }): Promise { + return this.impl.populateChecksums(options); + } + + dirtyBucketBatches(options: { minBucketChanges: number; minChangeRatio: number }): AsyncGenerator { + return this.impl.dirtyBucketBatches(options); + } + + dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { + return this.impl.dirtyBucketBatchForChecksums(options); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts new file mode 100644 index 000000000..473ad9666 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts @@ -0,0 +1,765 @@ +import { isMongoServerError, mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb'; +import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { + addChecksums, + InternalOpId, + isPartialChecksum, + PopulateChecksumCacheResults, + storage, + utils +} from '@powersync/service-core'; + +import { VersionedPowerSyncMongo } from '../db.js'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { + BucketDataDocumentV1, + BucketDataDocumentV3, + LEGACY_BUCKET_DATA_DEFINITION_ID, + TaggedBucketDataDocument, + bucketDataDocumentToTagged +} from '../models.js'; +import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; +import { cacheKey } from '../OperationBatch.js'; + +interface CurrentBucketState { + /** Bucket name */ + bucket: string; + definitionId: BucketDefinitionId; + /** + * Rows seen in the bucket, with the last op_id of each. + */ + seen: Map; + /** + * Estimated memory usage of the seen Map. + */ + trackingSize: number; + /** + * Last (lowest) seen op_id that is not a PUT. + */ + lastNotPut: InternalOpId | null; + /** + * Number of REMOVE/MOVE operations seen since lastNotPut. + */ + opsSincePut: number; + /** + * Incrementally-updated checksum, up to maxOpId. + */ + checksum: number; + /** + * Op count for the checksum. + */ + opCount: number; + /** + * Byte size of ops covered by the checksum. + */ + opBytes: number; +} + +type CompactBucketDataDocument = Pick< + TaggedBucketDataDocument, + '_id' | 'def' | 'op' | 'table' | 'row_id' | 'source_table' | 'source_key' | 'checksum' | 'target_op' +> & { + size: number | bigint; +}; + +type CompactClearBucketDataDocument = Pick; +type BucketDataCollectionDocument = BucketDataDocumentV1 | BucketDataDocumentV3; +type BucketDataClearProjection = { + _id: BucketDataCollectionDocument['_id']; + op: CompactClearBucketDataDocument['op']; + checksum: bigint; + target_op?: bigint | null; +}; + +type BucketStateProjection = { + _id: { b: string }; + estimate_since_compact?: { + count: number; + bytes: number | bigint; + }; + compacted_state?: { + count: number; + bytes: number | bigint | null; + }; +}; + +export interface MongoCompactOptions extends storage.CompactOptions {} + +const DEFAULT_CLEAR_BATCH_LIMIT = 5000; +const DEFAULT_MOVE_BATCH_LIMIT = 2000; +const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; +const DEFAULT_MIN_BUCKET_CHANGES = 10; +const DEFAULT_MIN_CHANGE_RATIO = 0.1; +const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; +/** This default is primarily for tests. */ +const DEFAULT_MEMORY_LIMIT_MB = 64; + +export interface DirtyBucket { + bucket: string; + definitionId: BucketDefinitionId | null; + estimatedCount: number; + dirtyRatio?: number; +} + +export abstract class BaseMongoCompactor { + protected updates: mongo.AnyBulkWriteOperation[] = []; + protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; + protected activeBucketDataCollection: mongo.Collection | null = null; + protected activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; + + protected readonly idLimitBytes: number; + protected readonly moveBatchLimit: number; + protected readonly moveBatchQueryLimit: number; + protected readonly clearBatchLimit: number; + protected readonly minBucketChanges: number; + protected readonly minChangeRatio: number; + protected readonly maxOpId: bigint; + protected readonly buckets: string[] | undefined; + protected readonly signal?: AbortSignal; + protected readonly group_id: number; + + constructor( + protected readonly storage: MongoSyncBucketStorage, + protected readonly db: VersionedPowerSyncMongo, + options: MongoCompactOptions + ) { + this.group_id = storage.group_id; + this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; + this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; + this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; + this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; + this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; + this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; + this.maxOpId = options.maxOpId ?? 0n; + this.buckets = options.compactBuckets; + this.signal = options.signal; + } + + /** + * Compact buckets by converting operations into MOVE and/or CLEAR operations. + * + * See /docs/compacting-operations.md for details. + */ + async compact() { + if (this.buckets) { + for (const bucket of this.buckets) { + // We can make this more efficient later on by iterating through the buckets in a single query. + // That makes batching more tricky, so we leave for later. + await this.compactSingleBucketRetried(bucket); + } + } else { + await this.compactDirtyBuckets(); + } + } + + /** + * Subset of compact, only populating checksums where relevant. + */ + async populateChecksums(options: { minBucketChanges: number }): Promise { + let count = 0; + while (true) { + this.signal?.throwIfAborted(); + const buckets = await this.dirtyBucketBatchForChecksums(options); + if (buckets.length == 0) { + break; + } + this.signal?.throwIfAborted(); + + const start = Date.now(); + // Filter batch by estimated bucket size, to reduce possibility of timeouts. + const checkBuckets: typeof buckets = []; + let totalCountEstimate = 0; + for (const bucket of buckets) { + checkBuckets.push(bucket); + totalCountEstimate += bucket.estimatedCount; + if (totalCountEstimate > 50_000) { + break; + } + } + logger.info( + `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` + ); + await this.updateChecksumsBatch(checkBuckets); + logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); + count += checkBuckets.length; + } + return { buckets: count }; + } + + protected async *dirtyBucketBatchesForCollection( + collection: mongo.Collection, + lastId: mongo.Document, + maxId: mongo.Document, + options: { + minBucketChanges: number; + minChangeRatio: number; + }, + getDefinitionId: (state: TBucketState) => BucketDefinitionId | null + ): AsyncGenerator { + while (true) { + // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline + // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria. + const [result] = await collection + .aggregate<{ + buckets: TBucketState[]; + cursor: Pick[]; + }>( + [ + { + $match: { + _id: { $gt: lastId, $lt: maxId } + } + }, + { + $sort: { _id: 1 } + }, + { + // Scan a fixed number of docs each query so sparse matches don't block progress. + $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE + }, + { + $facet: { + buckets: [ + { + $match: { + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + } + }, + { + $project: { + _id: 1, + estimate_since_compact: 1, + compacted_state: 1 + } + } + ], + // This is used for the next query. + cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] + } + } + ], + { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } + ) + .toArray(); + + const cursor = result?.cursor?.[0]; + if (cursor == null) { + break; + } + lastId = cursor._id as mongo.Document; + + const mapped = (result?.buckets ?? []).map((bucketState) => { + // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios. + // BigInt precision is not needed here since this is only an estimate. + const updatedCount = bucketState.estimate_since_compact?.count ?? 0; + const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; + const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); + const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; + const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; + const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; + return { + bucket: bucketState._id.b, + definitionId: getDefinitionId(bucketState), + estimatedCount: totalCount, + dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) + }; + }); + + yield mapped.filter( + (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio + ); + } + } + + protected async dirtyBucketBatchForChecksumsForCollection( + collection: mongo.Collection, + filter: mongo.Filter, + getDefinitionId: (state: mongo.WithId) => BucketDefinitionId | null + ): Promise { + const dirtyBuckets = await collection + .find(filter, { + projection: { + _id: 1, + estimate_since_compact: 1, + compacted_state: 1 + }, + sort: { + 'estimate_since_compact.count': -1 + }, + limit: 200, + maxTimeMS: MONGO_OPERATION_TIMEOUT_MS + }) + .toArray(); + + return dirtyBuckets.map((bucket) => ({ + bucket: bucket._id.b, + definitionId: getDefinitionId(bucket), + estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) + })); + } + + public abstract dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator; + + public abstract dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise; + + protected async compactDirtyBuckets() { + for await (const buckets of this.dirtyBucketBatches({ + minBucketChanges: this.minBucketChanges, + minChangeRatio: this.minChangeRatio + })) { + this.signal?.throwIfAborted(); + if (buckets.length == 0) { + continue; + } + + for (const { bucket, definitionId } of buckets) { + await this.compactSingleBucketRetried(bucket, definitionId); + } + } + } + + /** + * Compaction for a single bucket, with retries on failure. + * + * This covers against occasional network or other database errors during a long compact job. + */ + protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { + let retryCount = 0; + while (true) { + try { + await this.compactSingleBucket(bucket, definitionId); + break; + } catch (e) { + if (retryCount < 3 && isMongoServerError(e)) { + logger.warn(`Error compacting bucket ${bucket}, retrying...`, e); + retryCount++; + await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount)); + } else { + throw e; + } + } + } + } + + protected async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { + const idLimitBytes = this.idLimitBytes; + const bucketCollection = await this.getBucketDataCollection(bucket, definitionId); + if (bucketCollection == null) { + return; + } + this.activeBucketDataCollection = bucketCollection.collection; + this.activeBucketDefinitionId = bucketCollection.definitionId; + try { + const currentState: CurrentBucketState = { + bucket, + definitionId: bucketCollection.definitionId, + seen: new Map(), + trackingSize: 0, + lastNotPut: null, + opsSincePut: 0, + checksum: 0, + opCount: 0, + opBytes: 0 + }; + + // Constant lower bound. + const lowerBound = this.bucketDataKey(bucket, new mongo.MinKey() as any); + // Upper bound is adjusted for each batch. + let upperBound = this.bucketDataKey(bucket, new mongo.MaxKey() as any); + + while (true) { + this.signal?.throwIfAborted(); + + // Query one batch at a time, to avoid cursor timeouts. + const pipeline = [ + { + $match: { + _id: { + $gte: lowerBound, + $lt: upperBound + }, + // Workaround for a clustered collection bug where the $lt operator may include upperBound. + // https://jira.mongodb.org/browse/SERVER-121822 + '_id.o': { $lt: upperBound.o } + } + }, + { $sort: { _id: -1 } }, + { $limit: this.moveBatchQueryLimit }, + { + $project: { + _id: 1, + op: 1, + table: 1, + row_id: 1, + source_table: 1, + source_key: 1, + checksum: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ]; + + const cursor = bucketCollection.collection.aggregate( + pipeline, + { + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: this.moveBatchQueryLimit + 1 + } + ); + // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. + // Instead, we load up to the limit. + const rawBatch = await cursor.toArray(); + const batch = rawBatch.map((document) => this.tagBucketDataDocument(document, bucketCollection.definitionId)); + + if (batch.length == 0) { + // We've reached the end. + break; + } + + // Reuse the exact collection _id value from Mongo for the next bound. + upperBound = rawBatch[rawBatch.length - 1]._id; + + for (const doc of batch) { + if (doc._id.o > this.maxOpId) { + continue; + } + + currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum)); + currentState.opCount += 1; + + let isPersistentPut = doc.op == 'PUT'; + + currentState.opBytes += Number(doc.size); + if (doc.op == 'REMOVE' || doc.op == 'PUT') { + const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; + const targetOp = currentState.seen.get(key); + if (targetOp) { + // Will convert to MOVE, so don't count as PUT. + isPersistentPut = false; + + this.updates.push({ + updateOne: { + filter: { _id: this.bucketDataKey(doc._id.b, doc._id.o) }, + update: { + $set: { + op: 'MOVE', + target_op: targetOp + }, + $unset: { + source_table: 1, + source_key: 1, + table: 1, + row_id: 1, + data: 1 + } + } + } + }); + + // TODO: better estimate for this. + currentState.opBytes += 200 - Number(doc.size); + } else if (currentState.trackingSize < idLimitBytes) { + // flatstr reduces the memory usage by flattening the string. + currentState.seen.set(utils.flatstr(key), doc._id.o); + // length + 16 for the string + // 24 for the bigint + // 50 for map overhead + // 50 for additional overhead + currentState.trackingSize += key.length + 140; + } + } + + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (doc.op != 'CLEAR') { + if (currentState.lastNotPut == null) { + currentState.lastNotPut = doc._id.o; + } + currentState.opsSincePut += 1; + } + + if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { + await this.flush(); + } + } + + logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); + } + + // Free memory before clearing the bucket. + currentState.seen.clear(); + if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { + logger.info( + `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + // Need flush() before clear(). + await this.flush(); + await this.clearBucket(currentState); + } + + // Do this after clearBucket so we have accurate counts. + this.updateBucketChecksums(currentState); + // Need another flush after updateBucketChecksums(). + await this.flush(); + } finally { + this.activeBucketDataCollection = null; + this.activeBucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; + } + } + + protected updateBucketChecksums(state: CurrentBucketState) { + if (state.opCount < 0) { + throw new ServiceAssertionError( + `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` + ); + } + this.bucketStateUpdates.push({ + updateOne: { + filter: this.bucketStateFilter(state.bucket, state.definitionId), + update: { + $set: { + compacted_state: { + op_id: this.maxOpId, + count: state.opCount, + checksum: BigInt(state.checksum), + bytes: state.opBytes + }, + estimate_since_compact: { + // There could have been a whole bunch of new operations added to the bucket while compacting, + // which we don't currently cater for. We could potentially query for that, but that adds overhead. + count: 0, + bytes: 0 + } + } + }, + // We generally expect this to have been created before. + // We don't create new ones here, to avoid issues with the unique index on bucket_updates. + upsert: false + } + }); + } + + protected async flush() { + if (this.updates.length > 0) { + logger.info(`Compacting ${this.updates.length} ops`); + if (this.activeBucketDataCollection == null) { + throw new ServiceAssertionError('No bucket_data collection selected for compaction'); + } + await this.activeBucketDataCollection.bulkWrite(this.updates, { + // Order is not important. Since checksums are not affected, these operations can happen in any order, + // and it's fine if the operations are partially applied. Each individual operation is atomic. + ordered: false + }); + this.updates = []; + } + if (this.bucketStateUpdates.length > 0) { + logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); + await this.flushBucketStateUpdates(); + this.bucketStateUpdates = []; + } + } + + /** + * Perform a CLEAR compact for a bucket. + * + * @param currentState tracks the last non-PUT op, which will be converted to CLEAR. + */ + protected async clearBucket(currentState: CurrentBucketState) { + const bucket = currentState.bucket; + const clearOp = currentState.lastNotPut!; + const bucketCollection = this.activeBucketDataCollection; + if (bucketCollection == null) { + throw new ServiceAssertionError('No bucket_data collection selected for compaction'); + } + + const opFilter = { + _id: { + $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), + $lte: this.bucketDataKey(bucket, clearOp) + } + }; + + const session = this.db.client.startSession(); + try { + let done = false; + while (!done) { + this.signal?.throwIfAborted(); + let opCountDiff = 0; + // Do the CLEAR operation in batches, with each batch a separate transaction. + // The state after each batch is fully consistent. + // We need a transaction per batch to make sure checksums stay consistent. + await session.withTransaction( + async () => { + const query = bucketCollection.find(opFilter as any, { + session, + sort: { _id: 1 }, + projection: { + _id: 1, + op: 1, + checksum: 1, + target_op: 1 + }, + limit: this.clearBatchLimit + }); + let checksum = 0; + let lastOp: CompactClearBucketDataDocument | null = null; + let targetOp: bigint | null = null; + let gotAnOp = false; + let numberOfOpsToClear = 0; + for await (const rawOp of query.stream()) { + const op = this.tagClearBucketDataDocument( + rawOp as unknown as BucketDataClearProjection, + this.activeBucketDefinitionId + ); + + if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') { + checksum = utils.addChecksums(checksum, Number(op.checksum)); + lastOp = op; + numberOfOpsToClear += 1; + if (op.op != 'CLEAR') { + gotAnOp = true; + } + if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) { + targetOp = op.target_op; + } + } else { + throw new ReplicationAssertionError( + `Unexpected ${op.op} operation at ${this.formatBucketDataKey(op._id as unknown as mongo.Document)}` + ); + } + } + if (!gotAnOp) { + done = true; + return; + } + + logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?._id.o}`); + await bucketCollection.deleteMany( + { + _id: { + $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), + $lte: this.bucketDataKey(lastOp!._id.b, lastOp!._id.o) + } + } as any, + { session } as any + ); + + await bucketCollection.insertOne( + this.collectionBucketDataDocument({ + def: this.activeBucketDefinitionId, + _id: lastOp!._id, + op: 'CLEAR', + checksum: BigInt(checksum), + data: null, + target_op: targetOp + }) as unknown as mongo.OptionalId, + { session } as any + ); + + opCountDiff = -numberOfOpsToClear + 1; + }, + { + writeConcern: { w: 'majority' }, + readConcern: { level: 'snapshot' } + } + ); + // Update outside the transaction, since the transaction can be retried multiple times. + currentState.opCount += opCountDiff; + } + } finally { + await session.endSession(); + } + } + + protected async updateChecksumsBatch(buckets: Pick[]) { + const checksums = await this.computeChecksumsForBuckets(buckets); + const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); + + for (const bucketChecksum of checksums.values()) { + if (isPartialChecksum(bucketChecksum)) { + // Should never happen since we don't specify `start`. + throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); + } + + this.bucketStateUpdates.push({ + updateOne: { + filter: this.bucketStateFilter( + bucketChecksum.bucket, + definitionIdByBucket.get(bucketChecksum.bucket) ?? null + ), + update: { + $set: { + compacted_state: { + op_id: this.maxOpId, + count: bucketChecksum.count, + checksum: BigInt(bucketChecksum.checksum), + bytes: null + }, + estimate_since_compact: { + count: 0, + bytes: 0 + } + } + }, + // We don't create new ones here - it gets tricky to get the last_op right with the unique index on + // bucket_updates. + upsert: false + } + }); + } + + await this.flush(); + } + + protected tagBucketDataDocument( + document: BucketDataCollectionDocument & { size: number | bigint }, + definitionId: BucketDefinitionId + ): CompactBucketDataDocument { + const tagged = bucketDataDocumentToTagged(document, definitionId); + return { + ...tagged, + size: document.size + }; + } + + protected tagClearBucketDataDocument( + document: BucketDataClearProjection, + definitionId: BucketDefinitionId + ): CompactClearBucketDataDocument { + return { + def: definitionId, + _id: { + b: document._id.b, + o: document._id.o + }, + op: document.op, + checksum: document.checksum, + target_op: document.target_op + }; + } + + protected formatBucketDataKey(key: mongo.Document) { + const bucket = (key.b ?? key._id?.b) as string | undefined; + const op = (key.o ?? key._id?.o) as bigint | undefined; + return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; + } + + protected abstract flushBucketStateUpdates(): Promise; + protected abstract computeChecksumsForBuckets( + buckets: Pick[] + ): Promise; + protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; + protected abstract bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document; + protected abstract getBucketDataCollection( + bucket: string, + definitionId: BucketDefinitionId | null + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null>; + protected abstract collectionBucketDataDocument( + document: TaggedBucketDataDocument + ): BucketDataDocumentV1 | BucketDataDocumentV3; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts similarity index 96% rename from modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts rename to modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 8a7ae38f0..af849e4a9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -4,17 +4,17 @@ import * as bson from 'bson'; import { Logger, logger as defaultLogger, ReplicationAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; -import { MongoIdSequence } from './MongoIdSequence.js'; -import { VersionedPowerSyncMongo } from './db.js'; -import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { BucketDefinitionId } from './BucketDefinitionMapping.js'; +import { MongoIdSequence } from '../MongoIdSequence.js'; +import { VersionedPowerSyncMongo } from '../db.js'; +import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { BucketStateDocumentV1, BucketStateDocumentV3, TaggedBucketParameterDocument, TaggedBucketDataDocument -} from './models.js'; -import { mongoTableId } from '../../utils/util.js'; +} from '../models.js'; +import { mongoTableId } from '../../../utils/util.js'; import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js'; /** diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts b/modules/module-mongodb-storage/src/storage/implementation/common/SourceRecordStore.ts similarity index 94% rename from modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts rename to modules/module-mongodb-storage/src/storage/implementation/common/SourceRecordStore.ts index a9fed1a36..0984a3632 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStore.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/SourceRecordStore.ts @@ -3,7 +3,7 @@ import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; +import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; export interface SourceRecordLookupEntry { sourceTableId: bson.ObjectId; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts similarity index 74% rename from modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts rename to modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts index 785b4300f..238e73d92 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts @@ -1,7 +1,7 @@ -import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; -import { SourceRecordStore } from './SourceRecordStore.js'; +import { MongoBucketBatch, MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; +import { SourceRecordStore } from '../common/SourceRecordStore.js'; import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js'; -import { PersistedBatch } from './PersistedBatch.js'; +import { PersistedBatch } from '../common/PersistedBatch.js'; import { PersistedBatchV1 } from './PersistedBatchV1.js'; export class MongoBucketBatchV1 extends MongoBucketBatch { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts new file mode 100644 index 000000000..791f3514b --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -0,0 +1,100 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { InternalOpId, storage } from '@powersync/service-core'; +import { + BucketDataDocumentV1, + BucketStateDocumentV1, + LEGACY_BUCKET_DATA_DEFINITION_ID, + TaggedBucketDataDocument, + taggedBucketDataDocumentToV1 +} from '../models.js'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { BaseMongoCompactor, DirtyBucket } from '../common/MongoCompactorBase.js'; + +export class MongoCompactorV1 extends BaseMongoCompactor { + public async *dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only scan buckets with changes. + // That works well if there are only a small number of dirty buckets, but it causes repeated rescans while data is + // still changing. We now iterate through all V1 bucket_state rows for the group and filter after projecting. + yield* this.dirtyBucketBatchesForCollection( + this.db.bucketStateV1, + { g: this.group_id, b: new mongo.MinKey() as any }, + { g: this.group_id, b: new mongo.MaxKey() as any }, + options, + () => null + ); + } + + public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets + // estimate_since_compact as it progresses. + return this.dirtyBucketBatchForChecksumsForCollection( + this.db.bucketStateV1, + { + '_id.g': this.group_id, + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + }, + () => null + ); + } + + protected async flushBucketStateUpdates(): Promise { + await this.db.bucketStateV1.bulkWrite( + this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], + { + ordered: false + } + ); + } + + protected async computeChecksumsForBuckets( + buckets: Pick[] + ): Promise { + return this.storage.checksums.computePartialChecksumsDirectV1( + buckets.map(({ bucket }) => ({ + bucket, + end: this.maxOpId + })) + ); + } + + protected bucketStateFilter(bucket: string, _definitionId: BucketDefinitionId | null): mongo.Document { + return { + _id: { + g: this.group_id, + b: bucket + } + }; + } + + protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document { + return { + g: this.group_id, + b: bucket, + o: opId as any + }; + } + + protected async getBucketDataCollection( + _bucket: string, + _definitionId: BucketDefinitionId | null + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { + return { + collection: this.db.v1_bucket_data as unknown as mongo.Collection, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID + }; + } + + protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentV1 { + return taggedBucketDataDocumentToV1(this.group_id, document); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts similarity index 96% rename from modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts rename to modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index fcdb8e0c6..b21e94890 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -4,13 +4,13 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; import * as bson from 'bson'; -import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; +import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; import { PersistedBatch, SaveBucketDataOptions, SaveParameterDataOptions, UpsertCurrentDataOptions -} from './PersistedBatch.js'; +} from '../common/PersistedBatch.js'; import { BucketParameterDocument, CurrentDataDocument, @@ -19,8 +19,8 @@ import { SourceKey, taggedBucketParameterDocumentToV1, taggedBucketDataDocumentToV1 -} from './models.js'; -import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; +} from '../models.js'; +import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; export class PersistedBatchV1 extends PersistedBatch { currentData: mongo.AnyBulkWriteOperation[] = []; diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts similarity index 94% rename from modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts rename to modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts index fcb80fd34..bca3188b8 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts @@ -2,16 +2,16 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import * as bson from 'bson'; -import { idPrefixFilter } from '../../utils/util.js'; -import { VersionedPowerSyncMongo } from './db.js'; -import { cacheKey } from './OperationBatch.js'; +import { idPrefixFilter } from '../../../utils/util.js'; +import { VersionedPowerSyncMongo } from '../db.js'; +import { cacheKey } from '../OperationBatch.js'; import { SourceRecordLookupEntry, SourceRecordLookupState, LoadedSourceRecord, SourceRecordStore -} from './SourceRecordStore.js'; -import { CurrentDataDocument, SourceKey } from './models.js'; +} from '../common/SourceRecordStore.js'; +import { CurrentDataDocument, SourceKey } from '../models.js'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; export class SourceRecordStoreV1 implements SourceRecordStore { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts similarity index 75% rename from modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts rename to modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts index b5eb2662f..ee5f77b19 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -1,7 +1,7 @@ -import { MongoBucketBatch, MongoBucketBatchOptions } from './MongoBucketBatch.js'; -import { SourceRecordStore } from './SourceRecordStore.js'; +import { MongoBucketBatch, MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; +import { SourceRecordStore } from '../common/SourceRecordStore.js'; import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js'; -import { PersistedBatch } from './PersistedBatch.js'; +import { PersistedBatch } from '../common/PersistedBatch.js'; import { PersistedBatchV3 } from './PersistedBatchV3.js'; export class MongoBucketBatchV3 extends MongoBucketBatch { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts new file mode 100644 index 000000000..7c4a0f85e --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -0,0 +1,117 @@ +import { MONGO_OPERATION_TIMEOUT_MS, mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { InternalOpId, storage } from '@powersync/service-core'; +import { + BucketDataDocumentV3, + BucketStateDocumentV3, + TaggedBucketDataDocument, + taggedBucketDataDocumentToV3 +} from '../models.js'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { BaseMongoCompactor, DirtyBucket } from '../common/MongoCompactorBase.js'; + +export class MongoCompactorV3 extends BaseMongoCompactor { + public async *dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + // Same scan strategy as V1, but with the V3 bucket_state key shape. + yield* this.dirtyBucketBatchesForCollection( + this.db.bucketStateV3(this.group_id), + { d: new mongo.MinKey() as any, b: new mongo.MinKey() as any }, + { d: new mongo.MaxKey() as any, b: new mongo.MaxKey() as any }, + options, + (bucketState) => (bucketState as BucketStateDocumentV3)._id.d + ); + } + + public async dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { + if (options.minBucketChanges <= 0) { + throw new ReplicationAssertionError('minBucketChanges must be >= 1'); + } + return this.dirtyBucketBatchForChecksumsForCollection( + this.db.bucketStateV3(this.group_id), + { + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + }, + (bucketState) => (bucketState as BucketStateDocumentV3)._id.d + ); + } + + protected async flushBucketStateUpdates(): Promise { + await this.db + .bucketStateV3(this.group_id) + .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { + ordered: false + }); + } + + protected async computeChecksumsForBuckets( + buckets: Pick[] + ): Promise { + return this.storage.checksums.computePartialChecksumsDirectV3( + buckets.map(({ bucket, definitionId }) => { + if (definitionId == null) { + throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`); + } + return { + bucket, + definitionId, + end: this.maxOpId + }; + }) + ); + } + + protected bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document { + if (definitionId == null) { + throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`); + } + return { + _id: { + d: definitionId, + b: bucket + } + }; + } + + protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document { + return { b: bucket, o: opId as any }; + } + + protected async getBucketDataCollection( + bucket: string, + definitionId: BucketDefinitionId | null + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { + if (definitionId != null) { + return { + collection: this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, + definitionId + }; + } + + // FIXME: This is slow. It is only used when compacting a single bucket without a known definition id. + for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { + const existing = await collection.findOne( + { '_id.b': bucket }, + { projection: { _id: 1 }, maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } + ); + if (existing != null) { + const resolvedDefinitionId = collection.collectionName.replace(`bucket_data_${this.group_id}_`, ''); + return { + collection: collection as unknown as mongo.Collection, + definitionId: resolvedDefinitionId + }; + } + } + + return null; + } + + protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentV3 { + return taggedBucketDataDocumentToV3(document); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterLookupV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts similarity index 88% rename from modules/module-mongodb-storage/src/storage/implementation/MongoParameterLookupV3.ts rename to modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts index 3742fbd5c..c4793b7b3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterLookupV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts @@ -1,7 +1,7 @@ import * as bson from 'bson'; import { deserializeParameterLookup } from '@powersync/service-core'; import { ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules'; -import { ParameterIndexId } from './BucketDefinitionMapping.js'; +import { ParameterIndexId } from '../BucketDefinitionMapping.js'; export function serializeParameterLookupV3(lookup: ScopedParameterLookup): bson.Binary { return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) })); diff --git a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts similarity index 97% rename from modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts rename to modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 7aba884ab..67ce6b25f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -3,15 +3,15 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage, utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; import * as bson from 'bson'; -import { mongoTableId, replicaIdToSubkey } from '../../utils/util.js'; -import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; -import { BucketDefinitionId } from './BucketDefinitionMapping.js'; +import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; +import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { PersistedBatch, SaveBucketDataOptions, SaveParameterDataOptions, UpsertCurrentDataOptions -} from './PersistedBatch.js'; +} from '../common/PersistedBatch.js'; import { BucketParameterDocumentV3, CurrentDataDocumentV3, @@ -19,7 +19,7 @@ import { taggedBucketParameterDocumentToV3, taggedBucketDataDocumentToV3, SourceTableDocumentV3 -} from './models.js'; +} from '../models.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; export class PersistedBatchV3 extends PersistedBatch { diff --git a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts similarity index 95% rename from modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts rename to modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts index 6baa9d795..9f5a5a03a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts @@ -4,12 +4,12 @@ import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import * as bson from 'bson'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; -import { retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; -import { VersionedPowerSyncMongo } from './db.js'; -import { cacheKey } from './OperationBatch.js'; -import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from './SourceRecordStore.js'; -import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js'; -import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; +import { VersionedPowerSyncMongo } from '../db.js'; +import { cacheKey } from '../OperationBatch.js'; +import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js'; +import { CurrentDataDocumentV3, SourceTableDocumentV3 } from '../models.js'; +import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; export class SourceRecordStoreV3 implements SourceRecordStore { diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index 75e1f323b..3f554fe98 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -8,7 +8,7 @@ export * from './implementation/MongoStorageProvider.js'; export * from './implementation/MongoSyncBucketStorage.js'; export * from './implementation/MongoSyncRulesLock.js'; export * from './implementation/OperationBatch.js'; -export * from './implementation/PersistedBatch.js'; +export * from './implementation/common/PersistedBatch.js'; export * from '../utils/util.js'; export * from './MongoBucketStorage.js'; export * from './MongoReportStorage.js'; diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 75a5f459d..8dc02e9ef 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -6,7 +6,7 @@ import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/MongoSyncBucketStorage.js'; -import { SourceRecordStoreV3 } from '../../src/storage/implementation/SourceRecordStoreV3.js'; +import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js'; import { CurrentBucketV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; From b16600b334ac3e0e5ef11d97ffd2a6d22cd94c1b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 15:24:49 +0200 Subject: [PATCH 54/93] Split MongoChecksums. --- .../storage/implementation/MongoChecksums.ts | 577 +----------------- .../implementation/common/MongoChecksums.ts | 78 +++ .../common/MongoChecksumsBase.ts | 363 +++++++++++ .../implementation/v1/MongoChecksumsV1.ts | 77 +++ .../implementation/v3/MongoChecksumsV3.ts | 107 ++++ 5 files changed, 626 insertions(+), 576 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index 46f4348da..05370c74a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -1,576 +1 @@ -import * as lib_mongo from '@powersync/lib-service-mongodb'; -import { mongo } from '@powersync/lib-service-mongodb'; -import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { - addPartialChecksums, - bson, - BucketChecksumRequest, - BucketChecksum, - ChecksumCache, - ChecksumMap, - FetchPartialBucketChecksum, - InternalOpId, - isPartialChecksum, - PartialChecksum, - PartialChecksumMap, - PartialOrFullChecksum -} from '@powersync/service-core'; -import { VersionedPowerSyncMongo } from './db.js'; -import { BucketDefinitionId, BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { StorageConfig } from './models.js'; - -export interface FetchPartialBucketChecksumV3 { - bucket: string; - definitionId: BucketDefinitionId; - start?: InternalOpId; - end: InternalOpId; -} - -export interface FetchPartialBucketChecksumByBucket { - bucket: string; - start?: InternalOpId; - end: InternalOpId; -} - -/** - * Checksum calculation options, primarily for tests. - */ -export interface MongoChecksumOptions { - /** - * How many buckets to process in a batch when calculating checksums. - */ - bucketBatchLimit?: number; - - /** - * Limit on the number of documents to calculate a checksum on at a time. - */ - operationBatchLimit?: number; - - storageConfig: StorageConfig; - mapping?: BucketDefinitionMapping; -} - -const DEFAULT_BUCKET_BATCH_LIMIT = 200; -const DEFAULT_OPERATION_BATCH_LIMIT = 50_000; - -/** - * Shared checksum query plumbing. - * - * General implementation flow is: - * 1. getChecksums() -> check cache for (partial) matches. If not found or partial match, query the remainder using computePartialChecksums(). - * 2. computePartialChecksums() -> query bucket_state for partial matches. Query the remainder using computePartialChecksumsDirect(). - * 3. computePartialChecksumsDirect() -> split into batches of 200 buckets at a time -> computePartialChecksumsInternal() - * 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time - */ -abstract class AbstractMongoChecksums { - private _cache: ChecksumCache | undefined; - private readonly storageConfig: StorageConfig; - - constructor( - protected readonly db: VersionedPowerSyncMongo, - protected readonly group_id: number, - protected readonly options: MongoChecksumOptions - ) { - this.storageConfig = options.storageConfig; - } - - /** - * Lazy-instantiated cache. - * - * This means the cache only allocates memory once it is used for the first time. - */ - private get cache(): ChecksumCache { - this._cache ??= new ChecksumCache({ - fetchChecksums: (batch) => { - return this.computePartialChecksums(batch); - } - }); - return this._cache; - } - - /** - * Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from - * the database (bucket_state + bucket_data). - */ - async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise { - return this.cache.getChecksumMap(checkpoint, buckets); - } - - clearCache() { - this.cache.clear(); - } - - /** - * Calculate (partial) checksums from bucket_state (pre-aggregated) and bucket_data (individual operations). - * - * Results are not cached here. This method is only called by {@link ChecksumCache.getChecksumMap}, - * which is responsible for caching its result. - * - * As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state - * can be slow. - */ - private async computePartialChecksums(batch: FetchPartialBucketChecksum[]): Promise { - if (batch.length == 0) { - return new Map(); - } - const preStates = await this.fetchPreStates(batch); - - const mappedRequests = batch.map((request) => { - let start = request.start; - if (start == null) { - const preState = preStates.get(request.bucket); - if (preState != null) { - start = preState.opId; - } - } - return { - ...request, - start - }; - }); - - const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests); - - return new Map( - batch.map((request) => { - const bucket = request.bucket; - // Could be null if this is either (1) a partial request, or (2) no compacted checksum was available - const preState = preStates.get(bucket); - // Could be null if we got no data - const partialChecksum = queriedChecksums.get(bucket); - const merged = addPartialChecksums(bucket, preState?.checksum ?? null, partialChecksum ?? null); - - return [bucket, merged]; - }) - ); - } - - /** - * Calculate (partial) checksums from the data collection directly, bypassing the cache and bucket_state. - * - * Can be used directly in cases where the cache should be bypassed, such as from a compact job. - * - * Internally, we do calculations in smaller batches of buckets as appropriate. - * - * For large buckets, this can be slow, but should not time out as the underlying queries are performed in - * smaller batches. - */ - public async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise { - // Limit the number of buckets we query for at a time. - const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT; - - if (batch.length <= bucketBatchLimit) { - // Single batch - no need for splitting the batch and merging results - return await this.computePartialChecksumsInternal(batch); - } - // Split the batch and merge results - let results = new Map(); - for (let i = 0; i < batch.length; i += bucketBatchLimit) { - const bucketBatch = batch.slice(i, i + bucketBatchLimit); - const batchResults = await this.computePartialChecksumsInternal(bucketBatch); - for (let r of batchResults.values()) { - results.set(r.bucket, r); - } - } - return results; - } - - /** - * Query a batch of checksums. - * - * We limit the number of operations that the query aggregates in each sub-batch, to avoid potential query timeouts. - * - * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this. - */ - protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise; - - protected abstract fetchPreStates( - batch: FetchPartialBucketChecksum[] - ): Promise>; - - protected async computePartialChecksumsForCollection( - batch: TRequest[], - collection: mongo.Collection, - createFilter: (request: TRequest) => any - ): Promise { - const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT; - - // Map requests by bucket. We adjust this as we get partial results. - let requests = new Map(); - for (let request of batch) { - requests.set(request.bucket, request); - } - - const partialChecksums = new Map(); - - while (requests.size > 0) { - const filters = Array.from(requests.values(), createFilter); - - // Historically, checksum may be stored as 'int' or 'double'. - // More recently, this should be a 'long'. - // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations. - const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' }; - - // Aggregate over a max of `batchLimit` operations at a time. - // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12. - // Then we'll do three batches: - // 1. Query: A[1-end], B[1-end], C[1-end] - // Returns: A[1-10], B[1-2] - // 2. Query: B[3-end], C[1-end] - // Returns: B[3-10], C[1-4] - // 3. Query: C[5-end] - // Returns: C[5-10] - const aggregate = await collection - .aggregate( - [ - { - $match: { - $or: filters - } - }, - // sort and limit _before_ grouping - { $sort: { _id: 1 } }, - { $limit: batchLimit }, - { - $group: { - _id: '$_id.b', - checksum_total: { $sum: checksumLong }, - count: { $sum: 1 }, - has_clear_op: { - $max: { - $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0] - } - }, - last_op: { $max: '$_id.o' } - } - }, - // Sort the aggregated results (100 max, so should be fast). - // This is important to identify which buckets we have partial data for. - { $sort: { _id: 1 } } - ], - { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS } - ) - .toArray() - .catch((e) => { - throw lib_mongo.mapQueryError(e, 'while reading checksums'); - }); - - let batchCount = 0; - let limitReached = false; - for (let doc of aggregate) { - const bucket = doc._id; - const checksum = checksumFromAggregate(doc); - - const existing = partialChecksums.get(bucket); - if (existing != null) { - partialChecksums.set(bucket, addPartialChecksums(bucket, existing, checksum)); - } else { - partialChecksums.set(bucket, checksum); - } - - batchCount += doc.count; - if (batchCount == batchLimit) { - // Limit reached. Request more in the next batch. - // Note that this only affects the _last_ bucket in a batch. - limitReached = true; - const req = requests.get(bucket); - requests.set(bucket, { - ...req!, - start: doc.last_op - }); - } else { - // All done for this bucket - requests.delete(bucket); - } - } - if (!limitReached) { - break; - } - } - - return new Map( - batch.map((request) => { - const bucket = request.bucket; - // Could be null if we got no data - let partialChecksum = partialChecksums.get(bucket); - if (partialChecksum == null) { - partialChecksum = { - bucket, - partialCount: 0, - partialChecksum: 0 - }; - } - if (request.start == null && isPartialChecksum(partialChecksum)) { - partialChecksum = { - bucket, - count: partialChecksum.partialCount, - checksum: partialChecksum.partialChecksum - }; - } - - return [bucket, partialChecksum]; - }) - ); - } -} - -class MongoChecksumsV1Impl extends AbstractMongoChecksums { - async computePartialChecksumsDirectByBucket( - batch: FetchPartialBucketChecksumByBucket[] - ): Promise { - return this.computePartialChecksumsForCollection( - batch, - this.db.bucket_data as unknown as mongo.Collection, - (request) => ({ - _id: { - $gt: { - g: this.group_id, - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - g: this.group_id, - b: request.bucket, - o: request.end - } - } - }) - ); - } - - protected async fetchPreStates( - batch: FetchPartialBucketChecksum[] - ): Promise> { - const preFilters = batch - .filter((request) => request.start == null) - .map((request) => ({ - _id: { - g: this.group_id, - b: request.bucket - }, - 'compacted_state.op_id': { $exists: true, $lte: request.end } - })); - - const preStates = new Map(); - if (preFilters.length == 0) { - return preStates; - } - - const states = await this.db.bucketStateV1 - .find({ - $or: preFilters - }) - .toArray(); - - for (const state of states) { - const compactedState = state.compacted_state!; - preStates.set(state._id.b, { - opId: compactedState.op_id, - checksum: { - bucket: state._id.b, - checksum: Number(compactedState.checksum), - count: compactedState.count - } - }); - } - - return preStates; - } - - protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { - return this.computePartialChecksumsDirectByBucket(batch); - } -} - -class MongoChecksumsV3Impl extends AbstractMongoChecksums { - constructor( - db: VersionedPowerSyncMongo, - group_id: number, - options: MongoChecksumOptions, - private readonly mapping: BucketDefinitionMapping - ) { - super(db, group_id, options); - } - - private normalizeBatch(batch: FetchPartialBucketChecksum[]): FetchPartialBucketChecksumV3[] { - return batch.map((request) => ({ - bucket: request.bucket, - definitionId: this.mapping.bucketSourceId(request.source), - start: request.start, - end: request.end - })); - } - - async computePartialChecksumsDirectByDefinition(batch: FetchPartialBucketChecksumV3[]): Promise { - const results = new Map(); - const requestsByDefinition = new Map(); - - for (const request of batch) { - const existing = requestsByDefinition.get(request.definitionId) ?? []; - existing.push(request); - requestsByDefinition.set(request.definitionId, existing); - } - - for (const [definitionId, requests] of requestsByDefinition.entries()) { - const groupResults = await this.computePartialChecksumsForCollection( - requests, - this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, - createV3BucketFilter - ); - for (const checksum of groupResults.values()) { - results.set(checksum.bucket, checksum); - } - } - - return new Map( - batch.map((request) => [request.bucket, results.get(request.bucket) ?? emptyChecksumForRequest(request)]) - ); - } - - protected async fetchPreStates( - batch: FetchPartialBucketChecksum[] - ): Promise> { - const preFilters = this.normalizeBatch(batch) - .filter((request) => request.start == null) - .map((request) => ({ - _id: { - d: request.definitionId, - b: request.bucket - }, - 'compacted_state.op_id': { $exists: true, $lte: request.end } - })); - - const preStates = new Map(); - if (preFilters.length == 0) { - return preStates; - } - - const states = await this.db - .bucketStateV3(this.group_id) - .find({ - $or: preFilters - }) - .toArray(); - - for (const state of states) { - const compactedState = state.compacted_state!; - preStates.set(state._id.b, { - opId: compactedState.op_id, - checksum: { - bucket: state._id.b, - checksum: Number(compactedState.checksum), - count: compactedState.count - } - }); - } - - return preStates; - } - - protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { - return this.computePartialChecksumsDirectByDefinition(this.normalizeBatch(batch)); - } -} - -/** - * Public checksum API. Delegates to a storage-version-specific implementation. - */ -export class MongoChecksums { - private readonly impl: AbstractMongoChecksums; - private readonly v3Impl: MongoChecksumsV3Impl | null; - private readonly v1Impl: MongoChecksumsV1Impl | null; - - constructor(db: VersionedPowerSyncMongo, group_id: number, options: MongoChecksumOptions) { - if (options.storageConfig.incrementalReprocessing) { - this.v3Impl = new MongoChecksumsV3Impl( - db, - group_id, - options, - options.mapping ?? - (() => { - throw new ServiceAssertionError('BucketDefinitionMapping is required for v3 MongoDB checksum queries'); - })() - ); - this.v1Impl = null; - this.impl = this.v3Impl; - } else { - this.v3Impl = null; - this.v1Impl = new MongoChecksumsV1Impl(db, group_id, options); - this.impl = this.v1Impl; - } - } - - async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise { - return this.impl.getChecksums(checkpoint, buckets); - } - - clearCache() { - this.impl.clearCache(); - } - - async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise { - return this.impl.computePartialChecksumsDirect(batch); - } - - async computePartialChecksumsDirectV1(batch: FetchPartialBucketChecksumByBucket[]): Promise { - if (this.v1Impl == null) { - throw new ServiceAssertionError('V1 checksum routing is only available when incrementalReprocessing is disabled'); - } - return this.v1Impl.computePartialChecksumsDirectByBucket(batch); - } - - async computePartialChecksumsDirectV3(batch: FetchPartialBucketChecksumV3[]): Promise { - if (this.v3Impl == null) { - throw new ServiceAssertionError('V3 checksum routing is only available when incrementalReprocessing is enabled'); - } - return this.v3Impl.computePartialChecksumsDirectByDefinition(batch); - } -} - -function createV3BucketFilter(request: Pick) { - return { - _id: { - $gt: { - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - b: request.bucket, - o: request.end - } - } - }; -} - -function emptyChecksumForRequest( - request: Pick -): PartialOrFullChecksum { - if (request.start == null) { - return { bucket: request.bucket, count: 0, checksum: 0 }; - } - return { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }; -} - -/** - * Convert output of the $group stage into a checksum. - */ -function checksumFromAggregate(doc: bson.Document): PartialOrFullChecksum { - const partialChecksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff; - const bucket = doc._id; - - if (doc.has_clear_op == 1) { - return { - // full checksum - replaces any previous one - bucket, - checksum: partialChecksum, - count: doc.count - } satisfies BucketChecksum; - } else { - return { - // partial checksum - is added to a previous one - bucket, - partialCount: doc.count, - partialChecksum - } satisfies PartialChecksum; - } -} +export * from './common/MongoChecksums.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts new file mode 100644 index 000000000..07ffc1e93 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts @@ -0,0 +1,78 @@ +import { + BucketChecksumRequest, + ChecksumMap, + FetchPartialBucketChecksum, + InternalOpId, + PartialChecksumMap +} from '@powersync/service-core'; +import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { VersionedPowerSyncMongo } from '../db.js'; +import { MongoChecksumsV1Impl } from '../v1/MongoChecksumsV1.js'; +import { MongoChecksumsV3Impl } from '../v3/MongoChecksumsV3.js'; +import { + AbstractMongoChecksums, + FetchPartialBucketChecksumByBucket, + FetchPartialBucketChecksumV3, + MongoChecksumOptions +} from './MongoChecksumsBase.js'; + +export { + FetchPartialBucketChecksumByBucket, + FetchPartialBucketChecksumV3, + MongoChecksumOptions +} from './MongoChecksumsBase.js'; + +/** + * Public checksum API. Delegates to a storage-version-specific implementation. + */ +export class MongoChecksums { + private readonly impl: AbstractMongoChecksums; + private readonly v3Impl: MongoChecksumsV3Impl | null; + private readonly v1Impl: MongoChecksumsV1Impl | null; + + constructor(db: VersionedPowerSyncMongo, group_id: number, options: MongoChecksumOptions) { + if (options.storageConfig.incrementalReprocessing) { + this.v3Impl = new MongoChecksumsV3Impl( + db, + group_id, + options, + options.mapping ?? + (() => { + throw new ServiceAssertionError('BucketDefinitionMapping is required for v3 MongoDB checksum queries'); + })() + ); + this.v1Impl = null; + this.impl = this.v3Impl; + } else { + this.v3Impl = null; + this.v1Impl = new MongoChecksumsV1Impl(db, group_id, options); + this.impl = this.v1Impl; + } + } + + async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise { + return this.impl.getChecksums(checkpoint, buckets); + } + + clearCache() { + this.impl.clearCache(); + } + + async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise { + return this.impl.computePartialChecksumsDirect(batch); + } + + async computePartialChecksumsDirectV1(batch: FetchPartialBucketChecksumByBucket[]): Promise { + if (this.v1Impl == null) { + throw new ServiceAssertionError('V1 checksum routing is only available when incrementalReprocessing is disabled'); + } + return this.v1Impl.computePartialChecksumsDirectByBucket(batch); + } + + async computePartialChecksumsDirectV3(batch: FetchPartialBucketChecksumV3[]): Promise { + if (this.v3Impl == null) { + throw new ServiceAssertionError('V3 checksum routing is only available when incrementalReprocessing is enabled'); + } + return this.v3Impl.computePartialChecksumsDirectByDefinition(batch); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts new file mode 100644 index 000000000..2a5a5bf79 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts @@ -0,0 +1,363 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { + addPartialChecksums, + bson, + BucketChecksumRequest, + BucketChecksum, + ChecksumCache, + ChecksumMap, + FetchPartialBucketChecksum, + InternalOpId, + isPartialChecksum, + PartialChecksum, + PartialChecksumMap, + PartialOrFullChecksum +} from '@powersync/service-core'; +import { VersionedPowerSyncMongo } from '../db.js'; +import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { StorageConfig } from '../models.js'; + +export interface FetchPartialBucketChecksumV3 { + bucket: string; + definitionId: BucketDefinitionId; + start?: InternalOpId; + end: InternalOpId; +} + +export interface FetchPartialBucketChecksumByBucket { + bucket: string; + start?: InternalOpId; + end: InternalOpId; +} + +/** + * Checksum calculation options, primarily for tests. + */ +export interface MongoChecksumOptions { + /** + * How many buckets to process in a batch when calculating checksums. + */ + bucketBatchLimit?: number; + + /** + * Limit on the number of documents to calculate a checksum on at a time. + */ + operationBatchLimit?: number; + + storageConfig: StorageConfig; + mapping?: BucketDefinitionMapping; +} + +const DEFAULT_BUCKET_BATCH_LIMIT = 200; +const DEFAULT_OPERATION_BATCH_LIMIT = 50_000; + +/** + * Shared checksum query plumbing. + * + * General implementation flow is: + * 1. getChecksums() -> check cache for (partial) matches. If not found or partial match, query the remainder using computePartialChecksums(). + * 2. computePartialChecksums() -> query bucket_state for partial matches. Query the remainder using computePartialChecksumsDirect(). + * 3. computePartialChecksumsDirect() -> split into batches of 200 buckets at a time -> computePartialChecksumsInternal() + * 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time + */ +export abstract class AbstractMongoChecksums { + private _cache: ChecksumCache | undefined; + private readonly storageConfig: StorageConfig; + + constructor( + protected readonly db: VersionedPowerSyncMongo, + protected readonly group_id: number, + protected readonly options: MongoChecksumOptions + ) { + this.storageConfig = options.storageConfig; + } + + /** + * Lazy-instantiated cache. + * + * This means the cache only allocates memory once it is used for the first time. + */ + private get cache(): ChecksumCache { + this._cache ??= new ChecksumCache({ + fetchChecksums: (batch) => { + return this.computePartialChecksums(batch); + } + }); + return this._cache; + } + + /** + * Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from + * the database (bucket_state + bucket_data). + */ + async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise { + return this.cache.getChecksumMap(checkpoint, buckets); + } + + clearCache() { + this.cache.clear(); + } + + /** + * Calculate (partial) checksums from bucket_state (pre-aggregated) and bucket_data (individual operations). + * + * Results are not cached here. This method is only called by {@link ChecksumCache.getChecksumMap}, + * which is responsible for caching its result. + * + * As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state + * can be slow. + */ + private async computePartialChecksums(batch: FetchPartialBucketChecksum[]): Promise { + if (batch.length == 0) { + return new Map(); + } + const preStates = await this.fetchPreStates(batch); + + const mappedRequests = batch.map((request) => { + let start = request.start; + if (start == null) { + const preState = preStates.get(request.bucket); + if (preState != null) { + start = preState.opId; + } + } + return { + ...request, + start + }; + }); + + const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests); + + return new Map( + batch.map((request) => { + const bucket = request.bucket; + // Could be null if this is either (1) a partial request, or (2) no compacted checksum was available + const preState = preStates.get(bucket); + // Could be null if we got no data + const partialChecksum = queriedChecksums.get(bucket); + const merged = addPartialChecksums(bucket, preState?.checksum ?? null, partialChecksum ?? null); + + return [bucket, merged]; + }) + ); + } + + /** + * Calculate (partial) checksums from the data collection directly, bypassing the cache and bucket_state. + * + * Can be used directly in cases where the cache should be bypassed, such as from a compact job. + * + * Internally, we do calculations in smaller batches of buckets as appropriate. + * + * For large buckets, this can be slow, but should not time out as the underlying queries are performed in + * smaller batches. + */ + public async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise { + // Limit the number of buckets we query for at a time. + const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT; + + if (batch.length <= bucketBatchLimit) { + // Single batch - no need for splitting the batch and merging results + return await this.computePartialChecksumsInternal(batch); + } + // Split the batch and merge results + let results = new Map(); + for (let i = 0; i < batch.length; i += bucketBatchLimit) { + const bucketBatch = batch.slice(i, i + bucketBatchLimit); + const batchResults = await this.computePartialChecksumsInternal(bucketBatch); + for (let r of batchResults.values()) { + results.set(r.bucket, r); + } + } + return results; + } + + /** + * Query a batch of checksums. + * + * We limit the number of operations that the query aggregates in each sub-batch, to avoid potential query timeouts. + * + * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this. + */ + protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise; + + protected abstract fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise>; + + protected async computePartialChecksumsForCollection( + batch: TRequest[], + collection: mongo.Collection, + createFilter: (request: TRequest) => any + ): Promise { + const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT; + + // Map requests by bucket. We adjust this as we get partial results. + let requests = new Map(); + for (let request of batch) { + requests.set(request.bucket, request); + } + + const partialChecksums = new Map(); + + while (requests.size > 0) { + const filters = Array.from(requests.values(), createFilter); + + // Historically, checksum may be stored as 'int' or 'double'. + // More recently, this should be a 'long'. + // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations. + const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' }; + + // Aggregate over a max of `batchLimit` operations at a time. + // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12. + // Then we'll do three batches: + // 1. Query: A[1-end], B[1-end], C[1-end] + // Returns: A[1-10], B[1-2] + // 2. Query: B[3-end], C[1-end] + // Returns: B[3-10], C[1-4] + // 3. Query: C[5-end] + // Returns: C[5-10] + const aggregate = await collection + .aggregate( + [ + { + $match: { + $or: filters + } + }, + // sort and limit _before_ grouping + { $sort: { _id: 1 } }, + { $limit: batchLimit }, + { + $group: { + _id: '$_id.b', + checksum_total: { $sum: checksumLong }, + count: { $sum: 1 }, + has_clear_op: { + $max: { + $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0] + } + }, + last_op: { $max: '$_id.o' } + } + }, + // Sort the aggregated results (100 max, so should be fast). + // This is important to identify which buckets we have partial data for. + { $sort: { _id: 1 } } + ], + { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS } + ) + .toArray() + .catch((e) => { + throw lib_mongo.mapQueryError(e, 'while reading checksums'); + }); + + let batchCount = 0; + let limitReached = false; + for (let doc of aggregate) { + const bucket = doc._id; + const checksum = checksumFromAggregate(doc); + + const existing = partialChecksums.get(bucket); + if (existing != null) { + partialChecksums.set(bucket, addPartialChecksums(bucket, existing, checksum)); + } else { + partialChecksums.set(bucket, checksum); + } + + batchCount += doc.count; + if (batchCount == batchLimit) { + // Limit reached. Request more in the next batch. + // Note that this only affects the _last_ bucket in a batch. + limitReached = true; + const req = requests.get(bucket); + requests.set(bucket, { + ...req!, + start: doc.last_op + }); + } else { + // All done for this bucket + requests.delete(bucket); + } + } + if (!limitReached) { + break; + } + } + + return new Map( + batch.map((request) => { + const bucket = request.bucket; + // Could be null if we got no data + let partialChecksum = partialChecksums.get(bucket); + if (partialChecksum == null) { + partialChecksum = { + bucket, + partialCount: 0, + partialChecksum: 0 + }; + } + if (request.start == null && isPartialChecksum(partialChecksum)) { + partialChecksum = { + bucket, + count: partialChecksum.partialCount, + checksum: partialChecksum.partialChecksum + }; + } + + return [bucket, partialChecksum]; + }) + ); + } +} + +export function createV3BucketFilter(request: Pick) { + return { + _id: { + $gt: { + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + b: request.bucket, + o: request.end + } + } + }; +} + +export function emptyChecksumForRequest( + request: Pick +): PartialOrFullChecksum { + if (request.start == null) { + return { bucket: request.bucket, count: 0, checksum: 0 }; + } + return { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }; +} + +/** + * Convert output of the $group stage into a checksum. + */ +function checksumFromAggregate(doc: bson.Document): PartialOrFullChecksum { + const partialChecksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff; + const bucket = doc._id; + + if (doc.has_clear_op == 1) { + return { + // full checksum - replaces any previous one + bucket, + checksum: partialChecksum, + count: doc.count + } satisfies BucketChecksum; + } else { + return { + // partial checksum - is added to a previous one + bucket, + partialCount: doc.count, + partialChecksum + } satisfies PartialChecksum; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts new file mode 100644 index 000000000..5bb2a6c03 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts @@ -0,0 +1,77 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { + bson, + BucketChecksum, + FetchPartialBucketChecksum, + InternalOpId, + PartialChecksumMap +} from '@powersync/service-core'; +import { AbstractMongoChecksums, FetchPartialBucketChecksumByBucket } from '../common/MongoChecksumsBase.js'; + +export class MongoChecksumsV1Impl extends AbstractMongoChecksums { + async computePartialChecksumsDirectByBucket( + batch: FetchPartialBucketChecksumByBucket[] + ): Promise { + return this.computePartialChecksumsForCollection( + batch, + this.db.bucket_data as unknown as mongo.Collection, + (request) => ({ + _id: { + $gt: { + g: this.group_id, + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + g: this.group_id, + b: request.bucket, + o: request.end + } + } + }) + ); + } + + protected async fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise> { + const preFilters = batch + .filter((request) => request.start == null) + .map((request) => ({ + _id: { + g: this.group_id, + b: request.bucket + }, + 'compacted_state.op_id': { $exists: true, $lte: request.end } + })); + + const preStates = new Map(); + if (preFilters.length == 0) { + return preStates; + } + + const states = await this.db.bucketStateV1 + .find({ + $or: preFilters + }) + .toArray(); + + for (const state of states) { + const compactedState = state.compacted_state!; + preStates.set(state._id.b, { + opId: compactedState.op_id, + checksum: { + bucket: state._id.b, + checksum: Number(compactedState.checksum), + count: compactedState.count + } + }); + } + + return preStates; + } + + protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + return this.computePartialChecksumsDirectByBucket(batch); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts new file mode 100644 index 000000000..77bd61781 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -0,0 +1,107 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { + BucketChecksum, + FetchPartialBucketChecksum, + InternalOpId, + PartialChecksumMap, + PartialOrFullChecksum +} from '@powersync/service-core'; +import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { + AbstractMongoChecksums, + createV3BucketFilter, + emptyChecksumForRequest, + FetchPartialBucketChecksumV3, + MongoChecksumOptions +} from '../common/MongoChecksumsBase.js'; +import { VersionedPowerSyncMongo } from '../db.js'; + +export class MongoChecksumsV3Impl extends AbstractMongoChecksums { + constructor( + db: VersionedPowerSyncMongo, + group_id: number, + options: MongoChecksumOptions, + private readonly mapping: BucketDefinitionMapping + ) { + super(db, group_id, options); + } + + private normalizeBatch(batch: FetchPartialBucketChecksum[]): FetchPartialBucketChecksumV3[] { + return batch.map((request) => ({ + bucket: request.bucket, + definitionId: this.mapping.bucketSourceId(request.source), + start: request.start, + end: request.end + })); + } + + async computePartialChecksumsDirectByDefinition(batch: FetchPartialBucketChecksumV3[]): Promise { + const results = new Map(); + const requestsByDefinition = new Map(); + + for (const request of batch) { + const existing = requestsByDefinition.get(request.definitionId) ?? []; + existing.push(request); + requestsByDefinition.set(request.definitionId, existing); + } + + for (const [definitionId, requests] of requestsByDefinition.entries()) { + const groupResults = await this.computePartialChecksumsForCollection( + requests, + this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, + createV3BucketFilter + ); + for (const checksum of groupResults.values()) { + results.set(checksum.bucket, checksum); + } + } + + return new Map( + batch.map((request) => [request.bucket, results.get(request.bucket) ?? emptyChecksumForRequest(request)]) + ); + } + + protected async fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise> { + const preFilters = this.normalizeBatch(batch) + .filter((request) => request.start == null) + .map((request) => ({ + _id: { + d: request.definitionId, + b: request.bucket + }, + 'compacted_state.op_id': { $exists: true, $lte: request.end } + })); + + const preStates = new Map(); + if (preFilters.length == 0) { + return preStates; + } + + const states = await this.db + .bucketStateV3(this.group_id) + .find({ + $or: preFilters + }) + .toArray(); + + for (const state of states) { + const compactedState = state.compacted_state!; + preStates.set(state._id.b, { + opId: compactedState.op_id, + checksum: { + bucket: state._id.b, + checksum: Number(compactedState.checksum), + count: compactedState.count + } + }); + } + + return preStates; + } + + protected async computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise { + return this.computePartialChecksumsDirectByDefinition(this.normalizeBatch(batch)); + } +} From 8b07884bbc8c12174ef12b91774938e93fdceed2 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 15:30:32 +0200 Subject: [PATCH 55/93] Split MongoParameterCompactor. --- .../implementation/MongoParameterCompactor.ts | 146 +----------------- .../common/MongoParameterCompactor.ts | 21 +++ .../common/MongoParameterCompactorBase.ts | 131 ++++++++++++++++ .../v1/MongoParameterCompactorV1.ts | 23 +++ .../v3/MongoParameterCompactorV3.ts | 21 +++ 5 files changed, 197 insertions(+), 145 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts index ffa909ae7..0537f0397 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts @@ -1,145 +1 @@ -import { mongo } from '@powersync/lib-service-mongodb'; -import { logger } from '@powersync/lib-services-framework'; -import { bson, CompactOptions, InternalOpId } from '@powersync/service-core'; -import { LRUCache } from 'lru-cache'; -import { VersionedPowerSyncMongo } from './db.js'; -import { BucketParameterDocument, BucketParameterDocumentV3 } from './models.js'; - -/** - * Compacts parameter lookup data (the bucket_parameters collection). - * - * This scans through the entire collection to find data to compact. - * - * For background, see the `/docs/parameters-lookups.md` file. - */ -export class MongoParameterCompactor { - constructor( - private db: VersionedPowerSyncMongo, - private group_id: number, - private checkpoint: InternalOpId, - private options: CompactOptions - ) {} - - async compact() { - logger.info(`Compacting parameters for sync config ${this.group_id} up to checkpoint ${this.checkpoint}`); - if (this.db.storageConfig.incrementalReprocessing) { - await this.compactV3(); - return; - } - await this.compactV1(); - } - - private async compactV1() { - await this.compactCollection(this.db.parameterIndexV1); - } - - private async compactV3() { - for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { - await this.compactCollection(collection.collection); - } - } - - private async compactCollection(collection: mongo.Collection) { - // This is the currently-active checkpoint. - // We do not remove any data that may be used by this checkpoint. - // snapshot queries ensure that if any clients are still using older checkpoints, they would - // not be affected by this compaction. - const checkpoint = this.checkpoint; - - // Index on {'key.g': 1, lookup: 1, _id: 1} - // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup) - // in MongoDB already. However, that risks running into cases where MongoDB needs to process - // very large amounts of data before returning results, which could lead to timeouts. - const cursor = collection.find( - this.db.storageConfig.incrementalReprocessing - ? {} - : { - 'key.g': this.group_id - }, - { - sort: { lookup: 1, _id: 1 }, - batchSize: 10_000, - projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 } - } - ); - - // The index doesn't cover sorting by key, so we keep our own cache of the last seen key. - let lastByKey = new LRUCache({ - max: this.options.compactParameterCacheLimit ?? 10_000 - }); - let removeIds: InternalOpId[] = []; - let removeDeleted: mongo.AnyBulkWriteOperation[] = []; - let checkedEntries = 0; - let checkedEntriesAtLastLog = 0; - let lastProgressLogTime = Date.now(); - - const flush = async (force: boolean) => { - if (removeIds.length >= 1000 || (force && removeIds.length > 0)) { - const results = await collection.deleteMany({ _id: { $in: removeIds } }); - logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`); - removeIds = []; - } - - if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) { - const results = await collection.bulkWrite(removeDeleted); - logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`); - removeDeleted = []; - } - }; - - while (await cursor.hasNext()) { - const batch = cursor.readBufferedDocuments(); - checkedEntries += batch.length; - const now = Date.now(); - if (now - lastProgressLogTime >= 60_000) { - const elapsedSeconds = (now - lastProgressLogTime) / 1000; - const rate = (checkedEntries - checkedEntriesAtLastLog) / elapsedSeconds; - logger.info(`Checked ${checkedEntries} parameter index entries for compaction (${rate.toFixed(1)} entries/s)`); - lastProgressLogTime = now; - checkedEntriesAtLastLog = checkedEntries; - } - - for (let doc of batch) { - if (doc._id >= checkpoint) { - continue; - } - const uniqueKey = ( - bson.serialize({ - k: doc.key, - l: doc.lookup - }) as Buffer - ).toString('base64'); - const previous = lastByKey.get(uniqueKey); - if (previous != null && previous < doc._id) { - // We have a newer entry for the same key, so we can remove the old one. - removeIds.push(previous); - } - lastByKey.set(uniqueKey, doc._id); - - if (doc.bucket_parameters?.length == 0) { - // This is a delete operation, so we can remove it completely. - // For this we cannot remove the operation itself only: There is a possibility that - // there is still an earlier operation with the same key and lookup, that we don't have - // in the cache due to cache size limits. So we need to explicitly remove all earlier operations. - removeDeleted.push({ - deleteMany: { - filter: this.db.storageConfig.incrementalReprocessing - ? { lookup: doc.lookup, _id: { $lte: doc._id }, key: doc.key } - : { - 'key.g': (doc.key as BucketParameterDocument['key']).g, - lookup: doc.lookup, - _id: { $lte: doc._id }, - key: doc.key - } - } - }); - } - } - - await flush(false); - } - - await flush(true); - logger.info(`Parameter compaction completed for ${collection.collectionName}`); - } -} +export * from './common/MongoParameterCompactor.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts new file mode 100644 index 000000000..f2e2a1f17 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts @@ -0,0 +1,21 @@ +import { CompactOptions, InternalOpId } from '@powersync/service-core'; +import { VersionedPowerSyncMongo } from '../db.js'; +import { MongoParameterCompactorV1 } from '../v1/MongoParameterCompactorV1.js'; +import { MongoParameterCompactorV3 } from '../v3/MongoParameterCompactorV3.js'; +import { BaseMongoParameterCompactor } from './MongoParameterCompactorBase.js'; + +export class MongoParameterCompactor { + private readonly impl: BaseMongoParameterCompactor; + + constructor(db: VersionedPowerSyncMongo, group_id: number, checkpoint: InternalOpId, options: CompactOptions) { + if (db.storageConfig.incrementalReprocessing) { + this.impl = new MongoParameterCompactorV3(db, group_id, checkpoint, options); + } else { + this.impl = new MongoParameterCompactorV1(db, group_id, checkpoint, options); + } + } + + async compact() { + return this.impl.compact(); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts new file mode 100644 index 000000000..eedd0cc6b --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts @@ -0,0 +1,131 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { logger } from '@powersync/lib-services-framework'; +import { bson, CompactOptions, InternalOpId } from '@powersync/service-core'; +import { LRUCache } from 'lru-cache'; +import { VersionedPowerSyncMongo } from '../db.js'; + +type ParameterCompactionReadDocument = { + _id: InternalOpId; + key: mongo.Document; + lookup: unknown; + bucket_parameters?: unknown[] | null; +}; + +/** + * Compacts parameter lookup data (the bucket_parameters collection). + * + * This scans through the entire collection to find data to compact. + * + * For background, see the `/docs/parameters-lookups.md` file. + */ +export abstract class BaseMongoParameterCompactor { + constructor( + protected readonly db: VersionedPowerSyncMongo, + protected readonly group_id: number, + protected readonly checkpoint: InternalOpId, + protected readonly options: CompactOptions + ) {} + + async compact() { + logger.info(`Compacting parameters for sync config ${this.group_id} up to checkpoint ${this.checkpoint}`); + for (const collection of await this.getCollections()) { + await this.compactCollection(collection); + } + } + + protected abstract getCollections(): Promise[]>; + + protected abstract collectionFilter(): mongo.Document; + + protected abstract deleteFilter(doc: mongo.Document): mongo.Document; + + protected async compactCollection(collection: mongo.Collection) { + // This is the currently-active checkpoint. + // We do not remove any data that may be used by this checkpoint. + // snapshot queries ensure that if any clients are still using older checkpoints, they would + // not be affected by this compaction. + const checkpoint = this.checkpoint; + + // Index on {'key.g': 1, lookup: 1, _id: 1} + // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup) + // in MongoDB already. However, that risks running into cases where MongoDB needs to process + // very large amounts of data before returning results, which could lead to timeouts. + const cursor = collection.find(this.collectionFilter(), { + sort: { lookup: 1, _id: 1 }, + batchSize: 10_000, + projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 } + }); + + // The index doesn't cover sorting by key, so we keep our own cache of the last seen key. + let lastByKey = new LRUCache({ + max: this.options.compactParameterCacheLimit ?? 10_000 + }); + let removeIds: InternalOpId[] = []; + let removeDeleted: mongo.AnyBulkWriteOperation[] = []; + let checkedEntries = 0; + let checkedEntriesAtLastLog = 0; + let lastProgressLogTime = Date.now(); + + const flush = async (force: boolean) => { + if (removeIds.length >= 1000 || (force && removeIds.length > 0)) { + const results = await collection.deleteMany({ _id: { $in: removeIds } } as any); + logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`); + removeIds = []; + } + + if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) { + const results = await collection.bulkWrite(removeDeleted); + logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`); + removeDeleted = []; + } + }; + + while (await cursor.hasNext()) { + const batch = cursor.readBufferedDocuments() as unknown as ParameterCompactionReadDocument[]; + checkedEntries += batch.length; + const now = Date.now(); + if (now - lastProgressLogTime >= 60_000) { + const elapsedSeconds = (now - lastProgressLogTime) / 1000; + const rate = (checkedEntries - checkedEntriesAtLastLog) / elapsedSeconds; + logger.info(`Checked ${checkedEntries} parameter index entries for compaction (${rate.toFixed(1)} entries/s)`); + lastProgressLogTime = now; + checkedEntriesAtLastLog = checkedEntries; + } + + for (const doc of batch) { + if (doc._id >= checkpoint) { + continue; + } + const uniqueKey = ( + bson.serialize({ + k: doc.key, + l: doc.lookup + }) as Buffer + ).toString('base64'); + const previous = lastByKey.get(uniqueKey); + if (previous != null && previous < doc._id) { + // We have a newer entry for the same key, so we can remove the old one. + removeIds.push(previous); + } + lastByKey.set(uniqueKey, doc._id); + + if (doc.bucket_parameters?.length == 0) { + // This is a delete operation, so we can remove it completely. + // For this we cannot remove the operation itself only: There is a possibility that + // there is still an earlier operation with the same key and lookup, that we don't have + // in the cache due to cache size limits. So we need to explicitly remove all earlier operations. + removeDeleted.push({ + deleteMany: { + filter: this.deleteFilter(doc) + } + }); + } + } + + await flush(false); + } + + await flush(true); + logger.info(`Parameter compaction completed for ${collection.collectionName}`); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts new file mode 100644 index 000000000..224c9dd2c --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts @@ -0,0 +1,23 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { BaseMongoParameterCompactor } from '../common/MongoParameterCompactorBase.js'; + +export class MongoParameterCompactorV1 extends BaseMongoParameterCompactor { + protected async getCollections(): Promise[]> { + return [this.db.parameterIndexV1 as unknown as mongo.Collection]; + } + + protected collectionFilter(): mongo.Document { + return { + 'key.g': this.group_id + }; + } + + protected deleteFilter(doc: mongo.Document): mongo.Document { + return { + 'key.g': doc.key.g as number, + lookup: doc.lookup, + _id: { $lte: doc._id }, + key: doc.key + }; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts new file mode 100644 index 000000000..65999d718 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts @@ -0,0 +1,21 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { BaseMongoParameterCompactor } from '../common/MongoParameterCompactorBase.js'; + +export class MongoParameterCompactorV3 extends BaseMongoParameterCompactor { + protected async getCollections(): Promise[]> { + const collections = await this.db.listParameterIndexCollectionsV3(this.group_id); + return collections.map((collection) => collection.collection as unknown as mongo.Collection); + } + + protected collectionFilter(): mongo.Document { + return {}; + } + + protected deleteFilter(doc: mongo.Document): mongo.Document { + return { + lookup: doc.lookup, + _id: { $lte: doc._id }, + key: doc.key + }; + } +} From d61a280543ecb53f596bd82fd1353314777bf7c3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 15:38:09 +0200 Subject: [PATCH 56/93] Split methods in PersistedBatch. --- .../implementation/common/PersistedBatch.ts | 79 ++----------------- .../implementation/v1/PersistedBatchV1.ts | 34 ++++++++ .../implementation/v3/PersistedBatchV3.ts | 37 +++++++++ 3 files changed, 77 insertions(+), 73 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index af849e4a9..cb6003dfb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -2,18 +2,13 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { Logger, logger as defaultLogger, ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { MongoIdSequence } from '../MongoIdSequence.js'; import { VersionedPowerSyncMongo } from '../db.js'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { - BucketStateDocumentV1, - BucketStateDocumentV3, - TaggedBucketParameterDocument, - TaggedBucketDataDocument -} from '../models.js'; +import { TaggedBucketParameterDocument, TaggedBucketDataDocument } from '../models.js'; import { mongoTableId } from '../../../utils/util.js'; import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js'; @@ -121,6 +116,8 @@ export abstract class PersistedBatch { protected abstract flushCurrentData(session: mongo.ClientSession): Promise; + protected abstract flushBucketStates(session: mongo.ClientSession): Promise; + protected abstract resetCurrentData(): void; protected get bucketDataCount(): number { @@ -213,7 +210,6 @@ export abstract class PersistedBatch { } async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) { - const db = this.db; const startAt = performance.now(); let flushedSomething = false; if (this.bucketDataCount > 0) { @@ -231,17 +227,7 @@ export abstract class PersistedBatch { if (this.bucketStates.size > 0) { flushedSomething = true; - if (db.storageConfig.incrementalReprocessing) { - await db.bucketStateV3(this.group_id).bulkWrite(this.getBucketStateUpdatesV3(), { - session, - ordered: false - }); - } else { - await db.bucketStateV1.bulkWrite(this.getBucketStateUpdatesV1(), { - session, - ordered: false - }); - } + await this.flushBucketStates(session); } if (flushedSomething) { @@ -298,62 +284,9 @@ export abstract class PersistedBatch { return stats; } - - private getBucketStateUpdatesV1(): mongo.AnyBulkWriteOperation[] { - return Array.from(this.bucketStates.values()).map((state) => { - return { - updateOne: { - filter: { - _id: { - g: this.group_id, - b: state.bucket - } - }, - update: { - $set: { - last_op: state.lastOp - }, - $inc: { - 'estimate_since_compact.count': state.incrementCount, - 'estimate_since_compact.bytes': state.incrementBytes - } - }, - upsert: true - } - } satisfies mongo.AnyBulkWriteOperation; - }); - } - - private getBucketStateUpdatesV3(): mongo.AnyBulkWriteOperation[] { - return Array.from(this.bucketStates.values()).map((state) => { - if (state.definitionId == null) { - throw new ReplicationAssertionError('Expected bucket definition id when incrementalReprocessing is enabled'); - } - return { - updateOne: { - filter: { - _id: { - d: state.definitionId, - b: state.bucket - } - }, - update: { - $set: { - last_op: state.lastOp - }, - $inc: { - 'estimate_since_compact.count': state.incrementCount, - 'estimate_since_compact.bytes': state.incrementBytes - } - }, - upsert: true - } - } satisfies mongo.AnyBulkWriteOperation; - }); - } } -interface BucketStateUpdate { +export interface BucketStateUpdate { definitionId: BucketDefinitionId | null; bucket: string; lastOp: InternalOpId; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index b21e94890..9420e1b82 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -12,6 +12,7 @@ import { UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; import { + BucketStateDocumentV1, BucketParameterDocument, CurrentDataDocument, LEGACY_BUCKET_DATA_DEFINITION_ID, @@ -21,6 +22,7 @@ import { taggedBucketDataDocumentToV1 } from '../models.js'; import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; +import { BucketStateUpdate } from '../common/PersistedBatch.js'; export class PersistedBatchV1 extends PersistedBatch { currentData: mongo.AnyBulkWriteOperation[] = []; @@ -238,10 +240,42 @@ export class PersistedBatchV1 extends PersistedBatch { }); } + protected async flushBucketStates(session: mongo.ClientSession) { + await this.db.bucketStateV1.bulkWrite(this.getBucketStateUpdates(), { + session, + ordered: false + }); + } + protected resetCurrentData() { this.currentData = []; } + private getBucketStateUpdates(): mongo.AnyBulkWriteOperation[] { + return Array.from(this.bucketStates.values()).map((state: BucketStateUpdate) => { + return { + updateOne: { + filter: { + _id: { + g: this.group_id, + b: state.bucket + } + }, + update: { + $set: { + last_op: state.lastOp + }, + $inc: { + 'estimate_since_compact.count': state.incrementCount, + 'estimate_since_compact.bytes': state.incrementBytes + } + }, + upsert: true + } + } satisfies mongo.AnyBulkWriteOperation; + }); + } + private currentDataId(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId): SourceKey { return { g: this.group_id, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 67ce6b25f..6fb122ab7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -13,6 +13,7 @@ import { UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; import { + BucketStateDocumentV3, BucketParameterDocumentV3, CurrentDataDocumentV3, SourceTableKey, @@ -21,6 +22,7 @@ import { SourceTableDocumentV3 } from '../models.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; +import { BucketStateUpdate } from '../common/PersistedBatch.js'; export class PersistedBatchV3 extends PersistedBatch { currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation }[] = []; @@ -332,8 +334,43 @@ export class PersistedBatchV3 extends PersistedBatch { } } + protected async flushBucketStates(session: mongo.ClientSession) { + await this.db.bucketStateV3(this.group_id).bulkWrite(this.getBucketStateUpdates(), { + session, + ordered: false + }); + } + protected resetCurrentData() { this.currentData = []; this.sourceTablePendingDeletes.clear(); } + + private getBucketStateUpdates(): mongo.AnyBulkWriteOperation[] { + return Array.from(this.bucketStates.values()).map((state: BucketStateUpdate) => { + if (state.definitionId == null) { + throw new ReplicationAssertionError('Expected bucket definition id when incrementalReprocessing is enabled'); + } + return { + updateOne: { + filter: { + _id: { + d: state.definitionId, + b: state.bucket + } + }, + update: { + $set: { + last_op: state.lastOp + }, + $inc: { + 'estimate_since_compact.count': state.incrementCount, + 'estimate_since_compact.bytes': state.incrementBytes + } + }, + upsert: true + } + } satisfies mongo.AnyBulkWriteOperation; + }); + } } From 95871585b2ab4564c5e49d2e9991dc70a7f04886 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 15:42:22 +0200 Subject: [PATCH 57/93] Further split MongoBucketBatch. --- .../implementation/common/MongoBucketBatch.ts | 16 +++------------- .../implementation/v1/MongoBucketBatchV1.ts | 4 ++++ .../implementation/v3/MongoBucketBatchV3.ts | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts index 25f0c06c9..284872247 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts @@ -151,6 +151,8 @@ export abstract class MongoBucketBatch protected abstract get sourceRecordStore(): SourceRecordStore; + protected abstract cleanupDroppedSourceTables(sourceTables: storage.SourceTable[]): Promise; + async flush(options?: storage.BatchBucketFlushOptions): Promise { let result: storage.FlushedResult | null = null; // One flush may be split over multiple transactions. @@ -946,19 +948,7 @@ export abstract class MongoBucketBatch } }); - if (this.db.storageConfig.incrementalReprocessing) { - for (let table of sourceTables) { - await this.db - .sourceRecordsV3(this.group_id, mongoTableId(table.id)) - .drop() - .catch((error) => { - if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { - return; - } - throw error; - }); - } - } + await this.cleanupDroppedSourceTables(sourceTables); return result; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts index 238e73d92..8e10dce27 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts @@ -21,4 +21,8 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { protected get sourceRecordStore(): SourceRecordStore { return this.store; } + + protected async cleanupDroppedSourceTables(_sourceTables: import('@powersync/service-core').storage.SourceTable[]) { + // No-op for V1: source records live in a shared collection. + } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts index ee5f77b19..5d39bcc55 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -1,8 +1,11 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { storage } from '@powersync/service-core'; import { MongoBucketBatch, MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; import { SourceRecordStore } from '../common/SourceRecordStore.js'; import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; import { PersistedBatchV3 } from './PersistedBatchV3.js'; +import { mongoTableId } from '../../../utils/util.js'; export class MongoBucketBatchV3 extends MongoBucketBatch { private readonly store: SourceRecordStore; @@ -21,4 +24,18 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { protected get sourceRecordStore(): SourceRecordStore { return this.store; } + + protected async cleanupDroppedSourceTables(sourceTables: storage.SourceTable[]) { + for (const table of sourceTables) { + await this.db + .sourceRecordsV3(this.group_id, mongoTableId(table.id)) + .drop() + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } + } } From abe7cc348e9cd71f85592fd0c040ed15eeeb6940 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 16:02:01 +0200 Subject: [PATCH 58/93] Move out helpers for MongoSyncBucketStorage --- .../implementation/MongoSyncBucketStorage.ts | 687 +----------------- .../common/MongoSyncBucketStorageContext.ts | 15 + .../v1/MongoSyncBucketStorageV1.ts | 252 +++++++ .../v3/MongoSyncBucketStorageV3.ts | 351 +++++++++ 4 files changed, 649 insertions(+), 656 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 589386352..e57f33865 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -10,46 +10,25 @@ import { BroadcastIterable, CHECKPOINT_INVALIDATE_ALL, CheckpointChanges, - deserializeParameterLookup, GetCheckpointChangesOptions, InternalOpId, - internalToExternalOpId, maxLsn, mergeAsyncIterables, PopulateChecksumCacheOptions, PopulateChecksumCacheResults, - ProtocolOpId, ReplicationCheckpoint, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core'; -import { JSONBig } from '@powersync/service-jsonbig'; import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; -import { - idPrefixFilter, - mapOpEntry, - readSingleBatch, - retryOnMongoMaxTimeMSExpired, - setSessionSnapshotTime -} from '../../utils/util.js'; +import { idPrefixFilter, retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { VersionedPowerSyncMongo } from './db.js'; -import { - BucketDataDocumentV1, - BucketDataKeyV1, - BucketDataDocumentV3, - BucketParameterDocumentV3, - BucketStateDocument, - CommonSourceTableDocument, - LEGACY_BUCKET_DATA_DEFINITION_ID, - SourceKey, - StorageConfig, - bucketDataDocumentToTagged -} from './models.js'; +import { BucketDataKeyV1, BucketStateDocument, CommonSourceTableDocument, SourceKey, StorageConfig } from './models.js'; import { MongoBucketBatchV1 } from './v1/MongoBucketBatchV1.js'; import { MongoBucketBatchV3 } from './v3/MongoBucketBatchV3.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; @@ -57,7 +36,19 @@ import { MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; -import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './v3/MongoParameterLookupV3.js'; +import { + getBucketDataBatchV1, + getDataBucketChangesV1, + getParameterBucketChangesV1, + getParameterSetsV1 +} from './v1/MongoSyncBucketStorageV1.js'; +import { + getBucketDataBatchV3, + getDataBucketChangesV3, + getParameterBucketChangesV3, + getParameterSetsV3 +} from './v3/MongoSyncBucketStorageV3.js'; +import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js'; export interface MongoSyncBucketStorageOptions { checksumOptions?: Omit; @@ -116,6 +107,14 @@ export class MongoSyncBucketStorage return this.sync_rules.mapping; } + private get versionContext(): MongoSyncBucketStorageContext { + return { + db: this.db, + group_id: this.group_id, + mapping: this.mapping + }; + } + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { this.writeCheckpointAPI.setWriteCheckpointMode(mode); } @@ -406,167 +405,9 @@ export class MongoSyncBucketStorage lookups: ScopedParameterLookup[] ): Promise { if (this.db.storageConfig.incrementalReprocessing) { - return this.getParameterSetsV3(checkpoint, lookups); + return getParameterSetsV3(this.versionContext, checkpoint, lookups); } - return this.getParameterSetsV1(checkpoint, lookups); - } - - private async getParameterSetsV1( - checkpoint: MongoReplicationCheckpoint, - lookups: ScopedParameterLookup[] - ): Promise { - return this.db.client.withSession({ snapshot: true }, async (session) => { - // Set the session's snapshot time to the checkpoint's snapshot time. - // An alternative would be to create the session when the checkpoint is created, but managing - // the session lifetime would become more complex. - // Starting and ending sessions are cheap (synchronous when no transactions are used), - // so this should be fine. - // This is a roundabout way of setting {readConcern: {atClusterTime: clusterTime}}, since - // that is not exposed directly by the driver. - // Future versions of the driver may change the snapshotTime behavior, so we need tests to - // validate that this works as expected. We test this in the compacting tests. - setSessionSnapshotTime(session, checkpoint.snapshotTime); - const lookupFilter = lookups.map((lookup) => { - return storage.serializeLookup(lookup); - }); - // This query does not use indexes super efficiently, apart from the lookup filter. - // From some experimentation I could do individual lookups more efficient using an index - // on {'key.g': 1, lookup: 1, 'key.t': 1, 'key.k': 1, _id: -1}, - // but could not do the same using $group. - // For now, just rely on compacting to remove extraneous data. - // For a description of the data format, see the `/docs/parameters-lookups.md` file. - const rows = await this.db.parameterIndexV1 - .aggregate( - [ - { - $match: { - 'key.g': this.group_id, - lookup: { $in: lookupFilter }, - _id: { $lte: checkpoint.checkpoint } - } - }, - { - $sort: { - _id: -1 - } - }, - { - $group: { - _id: { key: '$key', lookup: '$lookup' }, - bucket_parameters: { - $first: '$bucket_parameters' - } - } - } - ], - { - session, - readConcern: 'snapshot', - // Limit the time for the operation to complete, to avoid getting connection timeouts - maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS - } - ) - .toArray() - .catch((e) => { - throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); - }); - const groupedParameters = rows.map((row) => { - return row.bucket_parameters; - }); - return groupedParameters.flat(); - }); - } - - private async getParameterSetsV3( - checkpoint: MongoReplicationCheckpoint, - lookups: ScopedParameterLookup[] - ): Promise { - return this.db.client.withSession({ snapshot: true }, async (session) => { - setSessionSnapshotTime(session, checkpoint.snapshotTime); - - // Conceptually we do each lookup separately as an aggregation pipeline. We then - // use $unionWith to combine it all into a single operation. - // This helps to: - // 1. Handle different collections in the same query (although this may not be common in practice). - // 2. Efficiently use the index to get the first item grouped by {lookup, key}. - // The index is on { lookup: 1, key: 1, _id: -1 }. - - const buildLookupPipeline = ( - lookup: ScopedParameterLookup - ): { - collection: mongo.Collection; - pipeline: mongo.Document[]; - } => { - const indexId = lookup.indexId; - const collection = this.db.parameterIndexV3(this.group_id, indexId); - const lookupFilter = serializeParameterLookupV3(lookup); - return { - collection, - pipeline: [ - { - $match: { - lookup: lookupFilter, - _id: { $lte: checkpoint.checkpoint } - } - }, - { - $sort: { - key: 1, - _id: -1 - } - }, - { - $group: { - _id: { - key: '$key' - }, - bucket_parameters: { - $first: '$bucket_parameters' - } - } - }, - { - $project: { - _id: 0, - bucket_parameters: 1 - } - } - ] - }; - }; - - const [firstLookup, ...remainingLookups] = lookups; - const firstQuery = firstLookup == null ? null : buildLookupPipeline(firstLookup); - if (firstQuery == null) { - return []; - } - - const pipeline: mongo.Document[] = [ - ...firstQuery.pipeline, - ...remainingLookups.map((lookup) => { - const query = buildLookupPipeline(lookup); - return { - $unionWith: { - coll: query.collection.collectionName, - pipeline: query.pipeline - } - }; - }) - ]; - - const rows = await firstQuery.collection - .aggregate<{ bucket_parameters: SqliteJsonRow[] }>(pipeline, { - session, - readConcern: 'snapshot', - maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS - }) - .toArray() - .catch((e) => { - throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); - }); - - return rows.flatMap((row) => row.bucket_parameters); - }); + return getParameterSetsV1(this.versionContext, checkpoint, lookups); } async *getBucketDataBatch( @@ -575,304 +416,10 @@ export class MongoSyncBucketStorage options?: storage.BucketDataBatchOptions ): AsyncIterable { if (this.db.storageConfig.incrementalReprocessing) { - yield* this.getBucketDataBatchV3(checkpoint, dataBuckets, options); - return; - } - - if (dataBuckets.length == 0) { - return; - } - let filters: mongo.Filter[] = []; - const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start])); - - if (checkpoint == null) { - throw new ServiceAssertionError('checkpoint is null'); - } - const end = checkpoint; - for (let { bucket: name, start } of dataBuckets) { - filters.push({ - _id: { - $gt: { - g: this.group_id, - b: name, - o: start - }, - $lte: { - g: this.group_id, - b: name, - o: end as any - } - } - }); - } - - // Internal naming: - // We do a query for one "batch", which may consist of multiple "chunks". - // Each chunk is limited to single bucket, and is limited in length and size. - // There are also overall batch length and size limits. - - const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; - const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; - - const cursor = this.db.bucket_data.find( - { - $or: filters - }, - { - session: undefined, - sort: { _id: 1 }, - limit: batchLimit, - // Increase batch size above the default 101, so that we can fill an entire batch in - // one go. - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: batchLimit + 1, - // Raw mode is returns an array of Buffer instead of parsed documents. - // We use it so that: - // 1. We can calculate the document size accurately without serializing again. - // 2. We can delay parsing the results until it's needed. - // We manually use bson.deserialize below - raw: true, - - // Limit the time for the operation to complete, to avoid getting connection timeouts - maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS - } - ) as unknown as mongo.FindCursor; - - // We want to limit results to a single batch to avoid high memory usage. - // This approach uses MongoDB's batch limits to limit the data here, which limits - // to the lower of the batch count and size limits. - // This is similar to using `singleBatch: true` in the find options, but allows - // detecting "hasMore". - let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => { - throw lib_mongo.mapQueryError(e, 'while reading bucket data'); - }); - if (data.length == batchLimit) { - // Limit reached - could have more data, despite the cursor being drained. - batchHasMore = true; - } - - let chunkSizeBytes = 0; - let currentChunk: utils.SyncBucketData | null = null; - let targetOp: InternalOpId | null = null; - - // Ordered by _id, meaning buckets are grouped together - for (let rawData of data) { - const row = bucketDataDocumentToTagged( - bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV1, - LEGACY_BUCKET_DATA_DEFINITION_ID - ); - const bucket = row._id.b; - - if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { - // We need to start a new chunk - let start: ProtocolOpId | undefined = undefined; - if (currentChunk != null) { - // There is an existing chunk we need to yield - if (currentChunk.bucket == bucket) { - // Current and new chunk have the same bucket, so need has_more on the current one. - // If currentChunk.bucket != bucket, then we reached the end of the previous bucket, - // and has_more = false in that case. - currentChunk.has_more = true; - start = currentChunk.next_after; - } - - const yieldChunk = currentChunk; - currentChunk = null; - chunkSizeBytes = 0; - yield { chunkData: yieldChunk, targetOp: targetOp }; - targetOp = null; - } - - if (start == null) { - const startOpId = bucketMap.get(bucket); - if (startOpId == null) { - throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`); - } - start = internalToExternalOpId(startOpId); - } - currentChunk = { - bucket, - after: start, - has_more: false, - data: [], - next_after: start - }; - targetOp = null; - } - - const entry = mapOpEntry(row); - - if (row.target_op != null) { - // MOVE, CLEAR - if (targetOp == null || row.target_op > targetOp) { - targetOp = row.target_op; - } - } - - currentChunk.data.push(entry); - currentChunk.next_after = entry.op_id; - - chunkSizeBytes += rawData.byteLength; - } - - if (currentChunk != null) { - const yieldChunk = currentChunk; - currentChunk = null; - // This is the final chunk in the batch. - // There may be more data if and only if the batch we retrieved isn't complete. - yieldChunk.has_more = batchHasMore; - yield { chunkData: yieldChunk, targetOp: targetOp }; - targetOp = null; - } - } - - /** - * Reads V3 bucket data across per-definition collections while presenting a single paginated - * stream to the caller. - * - * Unlike v1, the requested buckets may live in multiple collections. We therefore page through - * one definition group at a time. - * - * Important: as soon as any limit is hit for the current read, we stop and return control to - * the caller. We do not continue with the same group, and we do not move on to later groups. - * That keeps pagination boundaries predictable and matches the v1 behavior more closely. - */ - private async *getBucketDataBatchV3( - checkpoint: utils.InternalOpId, - dataBuckets: storage.BucketDataRequest[], - options?: storage.BucketDataBatchOptions - ): AsyncIterable { - if (dataBuckets.length == 0) { + yield* getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options); return; } - - if (checkpoint == null) { - throw new ServiceAssertionError('checkpoint is null'); - } - - const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; - const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; - const end = checkpoint; - let remainingLimit = batchLimit; - - const requestsByDefinition = new Map(); - for (const request of dataBuckets) { - const definitionId = this.sync_rules.mapping.bucketSourceId(request.source); - const requests = requestsByDefinition.get(definitionId) ?? []; - requests.push(request); - requestsByDefinition.set(definitionId, requests); - } - - const definitionGroups = Array.from(requestsByDefinition.entries()); - for (let groupIndex = 0; groupIndex < definitionGroups.length && remainingLimit > 0; groupIndex++) { - const [definitionId, requests] = definitionGroups[groupIndex]; - const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1; - const bucketMap = new Map(requests.map((request) => [request.bucket, request.start])); - const filters: mongo.Filter[] = Array.from(bucketMap.entries()).map(([bucket, start]) => ({ - _id: { - $gt: { - b: bucket, - o: start - }, - $lte: { - b: bucket, - o: end as any - } - } - })); - - const cursor = this.db.bucket_data_v3(this.group_id, definitionId).find( - { - $or: filters - }, - { - session: undefined, - sort: { _id: 1 }, - limit: remainingLimit, - batchSize: remainingLimit + 1, - raw: true, - maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS - } - ) as unknown as mongo.FindCursor; - - let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => { - throw lib_mongo.mapQueryError(e, 'while reading bucket data'); - }); - if (data.length == remainingLimit) { - batchHasMore = true; - } - if (data.length == 0) { - continue; - } - - remainingLimit -= data.length; - - let chunkSizeBytes = 0; - let currentChunk: utils.SyncBucketData | null = null; - let targetOp: InternalOpId | null = null; - - for (let rawData of data) { - const row = bucketDataDocumentToTagged( - bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3, - definitionId - ); - const bucket = row._id.b; - - if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { - let start: ProtocolOpId | undefined = undefined; - if (currentChunk != null) { - if (currentChunk.bucket == bucket) { - currentChunk.has_more = true; - start = currentChunk.next_after; - } - - const yieldChunk = currentChunk; - currentChunk = null; - chunkSizeBytes = 0; - yield { chunkData: yieldChunk, targetOp: targetOp }; - targetOp = null; - } - - if (start == null) { - const startOpId = bucketMap.get(bucket); - if (startOpId == null) { - throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`); - } - start = internalToExternalOpId(startOpId); - } - currentChunk = { - bucket, - after: start, - has_more: false, - data: [], - next_after: start - }; - } - - const entry = mapOpEntry(row); - if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) { - targetOp = row.target_op; - } - - currentChunk.data.push(entry); - currentChunk.next_after = entry.op_id; - chunkSizeBytes += rawData.byteLength; - } - - if (currentChunk != null) { - const yieldChunk = currentChunk; - // Stop after the current read if either: - // 1. MongoDB indicates more rows remain for this definition group, or - // 2. we exhausted the caller's overall document limit before later groups. - yieldChunk.has_more = batchHasMore || (remainingLimit <= 0 && hasLaterDefinitionGroups); - yield { chunkData: yieldChunk, targetOp: targetOp }; - } - - if (batchHasMore || remainingLimit <= 0) { - return; - } - } + yield* getBucketDataBatchV1(this.versionContext, checkpoint, dataBuckets, options); } async getChecksums( @@ -1331,190 +878,18 @@ export class MongoSyncBucketStorage options: GetCheckpointChangesOptions ): Promise> { if (this.db.storageConfig.incrementalReprocessing) { - return this.getDataBucketChangesV3(options); + return getDataBucketChangesV3(this.versionContext, options); } - return this.getDataBucketChangesV1(options); - } - - private async getDataBucketChangesV1( - options: GetCheckpointChangesOptions - ): Promise> { - const limit = 1000; - const bucketStateUpdates = await this.db.bucketStateV1 - .find( - { - // We have an index on (_id.g, last_op). - '_id.g': this.group_id, - last_op: { $gt: options.lastCheckpoint.checkpoint } - }, - { - projection: { - '_id.b': 1 - }, - limit: limit + 1, - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: limit + 2, - singleBatch: true - } - ) - .toArray(); - - const buckets = bucketStateUpdates.map((doc) => doc._id.b); - const invalidateDataBuckets = buckets.length > limit; - - return { - invalidateDataBuckets: invalidateDataBuckets, - updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets) - }; - } - - private async getDataBucketChangesV3( - options: GetCheckpointChangesOptions - ): Promise> { - const limit = 1000; - const bucketStateUpdates = await this.db - .bucketStateV3(this.group_id) - .aggregate<{ _id: string; last_op: bigint }>( - [ - { - $match: { - last_op: { $gt: options.lastCheckpoint.checkpoint } - } - }, - { - $group: { - _id: '$_id.b', - last_op: { $max: '$last_op' } - } - }, - { - $sort: { - last_op: 1 - } - }, - { - $limit: limit + 1 - } - ], - { maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS } - ) - .toArray(); - - const buckets = bucketStateUpdates.map((doc) => doc._id); - const invalidateDataBuckets = buckets.length > limit; - - return { - invalidateDataBuckets, - updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets) - }; + return getDataBucketChangesV1(this.versionContext, options); } private async getParameterBucketChanges( options: GetCheckpointChangesOptions ): Promise> { if (this.db.storageConfig.incrementalReprocessing) { - return this.getParameterBucketChangesV3(options); - } - return this.getParameterBucketChangesV1(options); - } - - private async getParameterBucketChangesV1( - options: GetCheckpointChangesOptions - ): Promise> { - const limit = 1000; - const parameterUpdates = await this.db.parameterIndexV1 - .find( - { - _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint }, - 'key.g': this.group_id - }, - { - projection: { - lookup: 1 - }, - limit: limit + 1, - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: limit + 2, - singleBatch: true - } - ) - .toArray(); - const invalidateParameterUpdates = parameterUpdates.length > limit; - - return { - invalidateParameterBuckets: invalidateParameterUpdates, - updatedParameterLookups: invalidateParameterUpdates - ? new Set() - : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup)))) - }; - } - - private async getParameterBucketChangesV3( - options: GetCheckpointChangesOptions - ): Promise> { - const limit = 1000; - const indexIds = this.mapping.allParameterIndexIds(); - const collections = indexIds.map((indexId) => ({ - indexId, - collection: this.db.parameterIndexV3(this.group_id, indexId) - })); - if (collections.length == 0) { - return { - invalidateParameterBuckets: false, - updatedParameterLookups: new Set() - }; + return getParameterBucketChangesV3(this.versionContext, options); } - const checkpointFilter = { - _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint } - }; - const pipelineForCollection = (indexId: string) => [ - { - $match: checkpointFilter - }, - { - $project: { - _id: 0, - lookup: 1, - indexId: { $literal: indexId } - } - } - ]; - const [firstCollection, ...remainingCollections] = collections; - const parameterUpdates = await firstCollection.collection - .aggregate<{ lookup: bson.Binary; indexId: string }>( - [ - ...pipelineForCollection(firstCollection.indexId), - ...remainingCollections.map((collection) => { - return { - $unionWith: { - coll: collection.collection.collectionName, - pipeline: pipelineForCollection(collection.indexId) - } - }; - }), - { - $limit: limit + 1 - } - ], - { - batchSize: limit + 2, - maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS - } - ) - .toArray(); - - const invalidateParameterUpdates = parameterUpdates.length > limit; - - return { - invalidateParameterBuckets: invalidateParameterUpdates, - updatedParameterLookups: invalidateParameterUpdates - ? new Set() - : new Set( - parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookupV3(p.lookup, p.indexId))) - ) - }; + return getParameterBucketChangesV1(this.versionContext, options); } // If we processed all connections together for each checkpoint, we could do a single lookup for all connections. diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts new file mode 100644 index 000000000..a6485b284 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts @@ -0,0 +1,15 @@ +import { InternalOpId } from '@powersync/service-core'; +import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { VersionedPowerSyncMongo } from '../db.js'; +import * as bson from 'bson'; + +export interface MongoSyncBucketStorageContext { + db: VersionedPowerSyncMongo; + group_id: number; + mapping: BucketDefinitionMapping; +} + +export interface MongoSyncBucketStorageCheckpoint { + checkpoint: InternalOpId; + snapshotTime: bson.Timestamp; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts new file mode 100644 index 000000000..b15fa5f32 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -0,0 +1,252 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { + CheckpointChanges, + deserializeParameterLookup, + GetCheckpointChangesOptions, + InternalOpId, + internalToExternalOpId, + ProtocolOpId, + storage, + utils +} from '@powersync/service-core'; +import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; +import { BucketDataDocumentV1, LEGACY_BUCKET_DATA_DEFINITION_ID, bucketDataDocumentToTagged } from '../models.js'; +import { + MongoSyncBucketStorageCheckpoint, + MongoSyncBucketStorageContext +} from '../common/MongoSyncBucketStorageContext.js'; + +export async function getParameterSetsV1( + ctx: MongoSyncBucketStorageContext, + checkpoint: MongoSyncBucketStorageCheckpoint, + lookups: ScopedParameterLookup[] +): Promise { + return ctx.db.client.withSession({ snapshot: true }, async (session) => { + setSessionSnapshotTime(session, checkpoint.snapshotTime); + const lookupFilter = lookups.map((lookup) => { + return storage.serializeLookup(lookup); + }); + const rows = await ctx.db.parameterIndexV1 + .aggregate( + [ + { + $match: { + 'key.g': ctx.group_id, + lookup: { $in: lookupFilter }, + _id: { $lte: checkpoint.checkpoint } + } + }, + { + $sort: { + _id: -1 + } + }, + { + $group: { + _id: { key: '$key', lookup: '$lookup' }, + bucket_parameters: { + $first: '$bucket_parameters' + } + } + } + ], + { + session, + readConcern: 'snapshot', + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) + .toArray() + .catch((e) => { + throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); + }); + const groupedParameters = rows.map((row) => { + return row.bucket_parameters; + }); + return groupedParameters.flat(); + }); +} + +export async function* getBucketDataBatchV1( + ctx: MongoSyncBucketStorageContext, + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions +): AsyncIterable { + if (dataBuckets.length == 0) { + return; + } + let filters: mongo.Filter[] = []; + const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start])); + + if (checkpoint == null) { + throw new Error('checkpoint is null'); + } + const end = checkpoint; + for (let { bucket: name, start } of dataBuckets) { + filters.push({ + _id: { + $gt: { + g: ctx.group_id, + b: name, + o: start + }, + $lte: { + g: ctx.group_id, + b: name, + o: end as any + } + } + }); + } + + const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + + const cursor = ctx.db.bucket_data.find( + { + $or: filters + }, + { + session: undefined, + sort: { _id: 1 }, + limit: batchLimit, + batchSize: batchLimit + 1, + raw: true, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) as unknown as mongo.FindCursor; + + let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => { + throw lib_mongo.mapQueryError(e, 'while reading bucket data'); + }); + if (data.length == batchLimit) { + batchHasMore = true; + } + + let chunkSizeBytes = 0; + let currentChunk: utils.SyncBucketData | null = null; + let targetOp: InternalOpId | null = null; + + for (let rawData of data) { + const row = bucketDataDocumentToTagged( + bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV1, + LEGACY_BUCKET_DATA_DEFINITION_ID + ); + const bucket = row._id.b; + + if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { + let start: ProtocolOpId | undefined = undefined; + if (currentChunk != null) { + if (currentChunk.bucket == bucket) { + currentChunk.has_more = true; + start = currentChunk.next_after; + } + + const yieldChunk = currentChunk; + currentChunk = null; + chunkSizeBytes = 0; + yield { chunkData: yieldChunk, targetOp: targetOp }; + targetOp = null; + } + + if (start == null) { + const startOpId = bucketMap.get(bucket); + if (startOpId == null) { + throw new Error(`data for unexpected bucket: ${bucket}`); + } + start = internalToExternalOpId(startOpId); + } + currentChunk = { + bucket, + after: start, + has_more: false, + data: [], + next_after: start + }; + targetOp = null; + } + + const entry = mapOpEntry(row); + + if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) { + targetOp = row.target_op; + } + + currentChunk.data.push(entry); + currentChunk.next_after = entry.op_id; + chunkSizeBytes += rawData.byteLength; + } + + if (currentChunk != null) { + const yieldChunk = currentChunk; + yieldChunk.has_more = batchHasMore; + yield { chunkData: yieldChunk, targetOp: targetOp }; + } +} + +export async function getDataBucketChangesV1( + ctx: MongoSyncBucketStorageContext, + options: GetCheckpointChangesOptions +): Promise> { + const limit = 1000; + const bucketStateUpdates = await ctx.db.bucketStateV1 + .find( + { + '_id.g': ctx.group_id, + last_op: { $gt: options.lastCheckpoint.checkpoint } + }, + { + projection: { + '_id.b': 1 + }, + limit: limit + 1, + batchSize: limit + 2, + singleBatch: true + } + ) + .toArray(); + + const buckets = bucketStateUpdates.map((doc) => doc._id.b); + const invalidateDataBuckets = buckets.length > limit; + + return { + invalidateDataBuckets, + updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets) + }; +} + +export async function getParameterBucketChangesV1( + ctx: MongoSyncBucketStorageContext, + options: GetCheckpointChangesOptions +): Promise> { + const limit = 1000; + const parameterUpdates = await ctx.db.parameterIndexV1 + .find( + { + _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint }, + 'key.g': ctx.group_id + }, + { + projection: { + lookup: 1 + }, + limit: limit + 1, + batchSize: limit + 2, + singleBatch: true + } + ) + .toArray(); + const invalidateParameterUpdates = parameterUpdates.length > limit; + + return { + invalidateParameterBuckets: invalidateParameterUpdates, + updatedParameterLookups: invalidateParameterUpdates + ? new Set() + : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup)))) + }; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts new file mode 100644 index 000000000..1a50a6766 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -0,0 +1,351 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { + CheckpointChanges, + GetCheckpointChangesOptions, + InternalOpId, + internalToExternalOpId, + ProtocolOpId, + storage, + utils +} from '@powersync/service-core'; +import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; +import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; +import { BucketDataDocumentV3, BucketParameterDocumentV3, bucketDataDocumentToTagged } from '../models.js'; +import { + MongoSyncBucketStorageCheckpoint, + MongoSyncBucketStorageContext +} from '../common/MongoSyncBucketStorageContext.js'; + +export async function getParameterSetsV3( + ctx: MongoSyncBucketStorageContext, + checkpoint: MongoSyncBucketStorageCheckpoint, + lookups: ScopedParameterLookup[] +): Promise { + return ctx.db.client.withSession({ snapshot: true }, async (session) => { + setSessionSnapshotTime(session, checkpoint.snapshotTime); + + const buildLookupPipeline = ( + lookup: ScopedParameterLookup + ): { + collection: mongo.Collection; + pipeline: mongo.Document[]; + } => { + const indexId = lookup.indexId; + const collection = ctx.db.parameterIndexV3(ctx.group_id, indexId); + const lookupFilter = serializeParameterLookupV3(lookup); + return { + collection, + pipeline: [ + { + $match: { + lookup: lookupFilter, + _id: { $lte: checkpoint.checkpoint } + } + }, + { + $sort: { + key: 1, + _id: -1 + } + }, + { + $group: { + _id: { + key: '$key' + }, + bucket_parameters: { + $first: '$bucket_parameters' + } + } + }, + { + $project: { + _id: 0, + bucket_parameters: 1 + } + } + ] + }; + }; + + const [firstLookup, ...remainingLookups] = lookups; + const firstQuery = firstLookup == null ? null : buildLookupPipeline(firstLookup); + if (firstQuery == null) { + return []; + } + + const pipeline: mongo.Document[] = [ + ...firstQuery.pipeline, + ...remainingLookups.map((lookup) => { + const query = buildLookupPipeline(lookup); + return { + $unionWith: { + coll: query.collection.collectionName, + pipeline: query.pipeline + } + }; + }) + ]; + + const rows = await firstQuery.collection + .aggregate<{ bucket_parameters: SqliteJsonRow[] }>(pipeline, { + session, + readConcern: 'snapshot', + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + }) + .toArray() + .catch((e) => { + throw lib_mongo.mapQueryError(e, 'while evaluating parameter queries'); + }); + + return rows.flatMap((row) => row.bucket_parameters); + }); +} + +export async function* getBucketDataBatchV3( + ctx: MongoSyncBucketStorageContext, + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions +): AsyncIterable { + if (dataBuckets.length == 0) { + return; + } + + if (checkpoint == null) { + throw new Error('checkpoint is null'); + } + + const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + const end = checkpoint; + let remainingLimit = batchLimit; + + const requestsByDefinition = new Map(); + for (const request of dataBuckets) { + const definitionId = ctx.mapping.bucketSourceId(request.source); + const requests = requestsByDefinition.get(definitionId) ?? []; + requests.push(request); + requestsByDefinition.set(definitionId, requests); + } + + const definitionGroups = Array.from(requestsByDefinition.entries()); + for (let groupIndex = 0; groupIndex < definitionGroups.length && remainingLimit > 0; groupIndex++) { + const [definitionId, requests] = definitionGroups[groupIndex]; + const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1; + const bucketMap = new Map(requests.map((request) => [request.bucket, request.start])); + const filters: mongo.Filter[] = Array.from(bucketMap.entries()).map(([bucket, start]) => ({ + _id: { + $gt: { + b: bucket, + o: start + }, + $lte: { + b: bucket, + o: end as any + } + } + })); + + const cursor = ctx.db.bucket_data_v3(ctx.group_id, definitionId).find( + { + $or: filters + }, + { + session: undefined, + sort: { _id: 1 }, + limit: remainingLimit, + batchSize: remainingLimit + 1, + raw: true, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) as unknown as mongo.FindCursor; + + let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => { + throw lib_mongo.mapQueryError(e, 'while reading bucket data'); + }); + if (data.length == remainingLimit) { + batchHasMore = true; + } + if (data.length == 0) { + continue; + } + + remainingLimit -= data.length; + + let chunkSizeBytes = 0; + let currentChunk: utils.SyncBucketData | null = null; + let targetOp: InternalOpId | null = null; + + for (let rawData of data) { + const row = bucketDataDocumentToTagged( + bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3, + definitionId + ); + const bucket = row._id.b; + + if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { + let start: ProtocolOpId | undefined = undefined; + if (currentChunk != null) { + if (currentChunk.bucket == bucket) { + currentChunk.has_more = true; + start = currentChunk.next_after; + } + + const yieldChunk = currentChunk; + currentChunk = null; + chunkSizeBytes = 0; + yield { chunkData: yieldChunk, targetOp: targetOp }; + targetOp = null; + } + + if (start == null) { + const startOpId = bucketMap.get(bucket); + if (startOpId == null) { + throw new Error(`data for unexpected bucket: ${bucket}`); + } + start = internalToExternalOpId(startOpId); + } + currentChunk = { + bucket, + after: start, + has_more: false, + data: [], + next_after: start + }; + } + + const entry = mapOpEntry(row); + if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) { + targetOp = row.target_op; + } + + currentChunk.data.push(entry); + currentChunk.next_after = entry.op_id; + chunkSizeBytes += rawData.byteLength; + } + + if (currentChunk != null) { + const yieldChunk = currentChunk; + yieldChunk.has_more = batchHasMore || (remainingLimit <= 0 && hasLaterDefinitionGroups); + yield { chunkData: yieldChunk, targetOp: targetOp }; + } + + if (batchHasMore || remainingLimit <= 0) { + return; + } + } +} + +export async function getDataBucketChangesV3( + ctx: MongoSyncBucketStorageContext, + options: GetCheckpointChangesOptions +): Promise> { + const limit = 1000; + const bucketStateUpdates = await ctx.db + .bucketStateV3(ctx.group_id) + .aggregate<{ _id: string; last_op: bigint }>( + [ + { + $match: { + last_op: { $gt: options.lastCheckpoint.checkpoint } + } + }, + { + $group: { + _id: '$_id.b', + last_op: { $max: '$last_op' } + } + }, + { + $sort: { + last_op: 1 + } + }, + { + $limit: limit + 1 + } + ], + { maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS } + ) + .toArray(); + + const buckets = bucketStateUpdates.map((doc) => doc._id); + const invalidateDataBuckets = buckets.length > limit; + + return { + invalidateDataBuckets, + updatedDataBuckets: invalidateDataBuckets ? new Set() : new Set(buckets) + }; +} + +export async function getParameterBucketChangesV3( + ctx: MongoSyncBucketStorageContext, + options: GetCheckpointChangesOptions +): Promise> { + const limit = 1000; + const indexIds = ctx.mapping.allParameterIndexIds(); + const collections = indexIds.map((indexId) => ({ + indexId, + collection: ctx.db.parameterIndexV3(ctx.group_id, indexId) + })); + if (collections.length == 0) { + return { + invalidateParameterBuckets: false, + updatedParameterLookups: new Set() + }; + } + const checkpointFilter = { + _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint } + }; + const pipelineForCollection = (indexId: string) => [ + { + $match: checkpointFilter + }, + { + $project: { + _id: 0, + lookup: 1, + indexId: { $literal: indexId } + } + } + ]; + const [firstCollection, ...remainingCollections] = collections; + const parameterUpdates = await firstCollection.collection + .aggregate<{ lookup: bson.Binary; indexId: string }>( + [ + ...pipelineForCollection(firstCollection.indexId), + ...remainingCollections.map((collection) => { + return { + $unionWith: { + coll: collection.collection.collectionName, + pipeline: pipelineForCollection(collection.indexId) + } + }; + }), + { + $limit: limit + 1 + } + ], + { + batchSize: limit + 2, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS + } + ) + .toArray(); + + const invalidateParameterUpdates = parameterUpdates.length > limit; + + return { + invalidateParameterBuckets: invalidateParameterUpdates, + updatedParameterLookups: invalidateParameterUpdates + ? new Set() + : new Set( + parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookupV3(p.lookup, p.indexId))) + ) + }; +} From 06bdb77edab43cf2add39402a3d3213753048291 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 16:53:13 +0200 Subject: [PATCH 59/93] Refactor MongoSyncBucketStorage. --- .../implementation/MongoSyncBucketStorage.ts | 969 ++---------------- .../common/MongoSyncBucketStorageBase.ts | 784 ++++++++++++++ .../v1/MongoSyncBucketStorageV1.ts | 130 ++- .../v3/MongoSyncBucketStorageV3.ts | 139 +++ 4 files changed, 1133 insertions(+), 889 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index e57f33865..dee84a6d8 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -1,969 +1,164 @@ -import * as lib_mongo from '@powersync/lib-service-mongodb'; -import { mongo } from '@powersync/lib-service-mongodb'; -import { - BaseObserver, - logger, - ReplicationAbortedError, - ServiceAssertionError -} from '@powersync/lib-services-framework'; +import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import { - BroadcastIterable, - CHECKPOINT_INVALIDATE_ALL, - CheckpointChanges, GetCheckpointChangesOptions, - InternalOpId, - maxLsn, - mergeAsyncIterables, PopulateChecksumCacheOptions, PopulateChecksumCacheResults, - ReplicationCheckpoint, storage, utils, WatchWriteCheckpointOptions } from '@powersync/service-core'; -import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { LRUCache } from 'lru-cache'; -import * as timers from 'timers/promises'; -import { idPrefixFilter, retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { VersionedPowerSyncMongo } from './db.js'; -import { BucketDataKeyV1, BucketStateDocument, CommonSourceTableDocument, SourceKey, StorageConfig } from './models.js'; -import { MongoBucketBatchV1 } from './v1/MongoBucketBatchV1.js'; -import { MongoBucketBatchV3 } from './v3/MongoBucketBatchV3.js'; -import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; -import { MongoCompactor } from './MongoCompactor.js'; -import { MongoParameterCompactor } from './MongoParameterCompactor.js'; +import { MongoChecksums } from './MongoChecksums.js'; import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; -import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; -import { - getBucketDataBatchV1, - getDataBucketChangesV1, - getParameterBucketChangesV1, - getParameterSetsV1 -} from './v1/MongoSyncBucketStorageV1.js'; -import { - getBucketDataBatchV3, - getDataBucketChangesV3, - getParameterBucketChangesV3, - getParameterSetsV3 -} from './v3/MongoSyncBucketStorageV3.js'; -import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js'; - -export interface MongoSyncBucketStorageOptions { - checksumOptions?: Omit; - storageConfig: StorageConfig; -} +import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorageBase.js'; +import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js'; +import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js'; -/** - * Only keep checkpoints around for a minute, before fetching a fresh one. - * - * The reason is that we keep a MongoDB snapshot reference (clusterTime) with the checkpoint, - * and they expire after 5 minutes by default. This is an issue if the checkpoint stream is idle, - * but new clients connect and use an outdated checkpoint snapshot for parameter queries. - * - * These will be filtered out for existing clients, so should not create significant overhead. - */ -const CHECKPOINT_TIMEOUT_MS = 60_000; - -export class MongoSyncBucketStorage - extends BaseObserver - implements storage.SyncRulesBucketStorage -{ - readonly db: VersionedPowerSyncMongo; - readonly checksums: MongoChecksums; - - private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined; - private writeCheckpointAPI: MongoWriteCheckpointAPI; - #storageInitialized = false; +export { MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorageBase.js'; + +export class MongoSyncBucketStorage implements storage.SyncRulesBucketStorage { + private readonly impl: BaseMongoSyncBucketStorage; constructor( - public readonly factory: MongoBucketStorage, - public readonly group_id: number, - private readonly sync_rules: MongoPersistedSyncRulesContent, - public readonly slot_name: string, + factory: MongoBucketStorage, + group_id: number, + sync_rules: MongoPersistedSyncRulesContent, + slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions ) { - super(); - this.db = factory.db.versioned(sync_rules.getStorageConfig()); - this.checksums = new MongoChecksums(this.db, this.group_id, { - ...options.checksumOptions, - storageConfig: options?.storageConfig, - mapping: sync_rules.mapping - }); - this.writeCheckpointAPI = new MongoWriteCheckpointAPI({ - db: this.db, - mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED, - sync_rules_id: group_id - }); + if (sync_rules.getStorageConfig().incrementalReprocessing) { + this.impl = new MongoSyncBucketStorageV3(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); + } else { + this.impl = new MongoSyncBucketStorageV1(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); + } } - get writeCheckpointMode() { - return this.writeCheckpointAPI.writeCheckpointMode; + get factory(): MongoBucketStorage { + return this.impl.factory; + } + + get group_id(): number { + return this.impl.group_id; + } + + get slot_name(): string { + return this.impl.slot_name; + } + + get db(): VersionedPowerSyncMongo { + return this.impl.db; + } + + get checksums(): MongoChecksums { + return this.impl.checksums; } get mapping() { - return this.sync_rules.mapping; + return this.impl.mapping; } - private get versionContext(): MongoSyncBucketStorageContext { - return { - db: this.db, - group_id: this.group_id, - mapping: this.mapping - }; + get writeCheckpointMode() { + return this.impl.writeCheckpointMode; + } + + registerListener(listener: Partial): () => void { + return this.impl.registerListener(listener); } setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { - this.writeCheckpointAPI.setWriteCheckpointMode(mode); + this.impl.setWriteCheckpointMode(mode); } createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { - return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint); + return this.impl.createManagedWriteCheckpoint(checkpoint); } lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { - return this.writeCheckpointAPI.lastWriteCheckpoint({ - ...filters, - sync_rules_id: this.group_id - }); + return this.impl.lastWriteCheckpoint(filters); } getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncRules { - const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; - /** - * Check if the cached sync rules, if present, had the same options. - * Parse sync rules if the options are different or if there is no cached value. - */ - if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { - this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options }; - } - - return this.parsedSyncRulesCache!.parsed; + return this.impl.getParsedSyncRules(options); } - async getCheckpoint(): Promise { - return (await this.getCheckpointInternal()) ?? new EmptyReplicationCheckpoint(); + getCheckpoint(): Promise { + return this.impl.getCheckpoint(); } - async getCheckpointInternal(): Promise { - return await this.db.client.withSession({ snapshot: true }, async (session) => { - const doc = await this.db.sync_rules.findOne( - { _id: this.group_id }, - { - session, - projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 } - } - ); - if (!doc?.snapshot_done || !['ACTIVE', 'ERRORED'].includes(doc.state)) { - // Sync rules not active - return null - return null; - } - - // Specifically using operationTime instead of clusterTime - // There are 3 fields in the response: - // 1. operationTime, not exposed for snapshot sessions (used for causal consistency) - // 2. clusterTime (used for connection management) - // 3. atClusterTime, which is session.snapshotTime - // We use atClusterTime, to match the driver's internal snapshot handling. - // There are cases where clusterTime > operationTime and atClusterTime, - // which could cause snapshot queries using this as the snapshotTime to timeout. - // This was specifically observed on MongoDB 6.0 and 7.0. - const snapshotTime = (session as any).snapshotTime as bson.Timestamp | undefined; - if (snapshotTime == null) { - throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()'); - } - return new MongoReplicationCheckpoint( - this, - // null/0n is a valid checkpoint in some cases, for example if the initial snapshot was empty - doc.last_checkpoint ?? 0n, - doc.last_checkpoint_lsn ?? null, - snapshotTime - ); - }); + getCheckpointInternal(): Promise { + return this.impl.getCheckpointInternal(); } - private async initializeStorage() { - if (this.#storageInitialized) { - return; - } - - await this.db.initializeStreamStorage(this.group_id); - - const mapping = this.sync_rules.mapping; - for (let source of mapping.allBucketDefinitionIds()) { - const collection = this.db.bucket_data_v3(this.group_id, source).collectionName; - await this.db.db - .createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }) - .catch((error) => { - if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') { - return; - } - throw error; - }); - } - for (let indexId of mapping.allParameterIndexIds()) { - await this.db.parameterIndexV3(this.group_id, indexId).createIndex( - { - lookup: 1, - key: 1, - _id: -1 - }, - { - name: 'lookup_op_id' - } - ); - } - this.#storageInitialized = true; - } - - async createWriter(options: storage.CreateWriterOptions): Promise { - await this.initializeStorage(); - - const doc = await this.db.sync_rules.findOne( - { - _id: this.group_id - }, - { projection: { last_checkpoint_lsn: 1, no_checkpoint_before: 1, keepalive_op: 1, snapshot_lsn: 1 } } - ); - const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null; - - const batchOptions = { - logger: options.logger, - db: this.db, - syncRules: this.sync_rules.parsed(options).hydratedSyncRules(), - mapping: this.sync_rules.mapping, - groupId: this.group_id, - slotName: this.slot_name, - lastCheckpointLsn: checkpoint_lsn, - resumeFromLsn: maxLsn(checkpoint_lsn, doc?.snapshot_lsn), - keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null, - storeCurrentData: options.storeCurrentData, - skipExistingRows: options.skipExistingRows ?? false, - markRecordUnavailable: options.markRecordUnavailable - }; - const writer = this.db.storageConfig.incrementalReprocessing - ? new MongoBucketBatchV3(batchOptions) - : new MongoBucketBatchV1(batchOptions); - this.iterateListeners((cb) => cb.batchStarted?.(writer)); - return writer; + createWriter(options: storage.CreateWriterOptions): Promise { + return this.impl.createWriter(options); } - /** - * @deprecated Use `createWriter()` with `await using` instead. - */ - async startBatch( + startBatch( options: storage.CreateWriterOptions, callback: (batch: storage.BucketStorageBatch) => Promise ): Promise { - await using writer = await this.createWriter(options); - await callback(writer); - await writer.flush(); - return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + return this.impl.startBatch(options, callback); } - async resolveTable(options: storage.ResolveTableOptions): Promise { - const { group_id, connection_id, connection_tag, entity_descriptor } = options; - - const { schema, name, objectId, replicaIdColumns } = entity_descriptor; - - const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({ - name: column.name, - type: column.type, - type_oid: column.typeId - })); - const mapping = this.sync_rules.mapping; - let result: storage.ResolveTableResult | null = null; - let initializeSourceRecordsFor: bson.ObjectId | null = null; - - const baseId: Partial = this.db.storageConfig.incrementalReprocessing - ? {} - : { group_id }; - await this.db.client.withSession(async (session) => { - const col = this.db.commonSourceTables(group_id); - let filter: Partial = { - ...baseId, - connection_id: connection_id, - schema_name: schema, - table_name: name, - replica_id_columns2: normalizedReplicaIdColumns - }; - - if (objectId != null) { - filter.relation_id = objectId; - } - let doc = await col.findOne(filter, { session }); - if (doc == null) { - const candidateSourceTable = new storage.SourceTable({ - id: new bson.ObjectId(), - connectionTag: connection_tag, - objectId: objectId, - schema: schema, - name: name, - replicaIdColumns: replicaIdColumns, - snapshotComplete: false - }); - const createDoc: CommonSourceTableDocument = { - _id: candidateSourceTable.id as bson.ObjectId, - ...(baseId as any), - connection_id: connection_id, - relation_id: objectId, - schema_name: schema, - table_name: name, - replica_id_columns: null, - replica_id_columns2: normalizedReplicaIdColumns, - snapshot_done: false, - snapshot_status: undefined - }; - if (this.db.storageConfig.incrementalReprocessing) { - const bucketDataSourceIds = options.sync_rules.definition.bucketDataSources - .filter((source) => source.tableSyncsData(candidateSourceTable)) - .map((source) => mapping.bucketSourceId(source)); - const parameterLookupSourceIds = options.sync_rules.definition.bucketParameterLookupSources - .filter((source) => source.tableSyncsParameters(candidateSourceTable)) - .map((source) => mapping.parameterLookupId(source)); - - Object.assign(createDoc, { - bucket_data_source_ids: bucketDataSourceIds, - parameter_lookup_source_ids: parameterLookupSourceIds - }); - } - doc = createDoc; - - await col.insertOne(doc, { session }); - if (this.db.storageConfig.incrementalReprocessing) { - initializeSourceRecordsFor = doc._id; - } - } - const sourceTable = new storage.SourceTable({ - id: doc._id, - connectionTag: connection_tag, - objectId: objectId, - schema: schema, - name: name, - replicaIdColumns: replicaIdColumns, - snapshotComplete: doc.snapshot_done ?? true - }); - sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); - sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); - sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable); - sourceTable.snapshotStatus = - doc.snapshot_status == null - ? undefined - : { - lastKey: doc.snapshot_status.last_key?.buffer ?? null, - totalEstimatedCount: doc.snapshot_status.total_estimated_count, - replicatedCount: doc.snapshot_status.replicated_count - }; - - let dropTables: storage.SourceTable[] = []; - // Detect tables that are either renamed, or have different replica_id_columns - let truncateFilter = [{ schema_name: schema, table_name: name }] as any[]; - if (objectId != null) { - // Only detect renames if the source uses relation ids. - truncateFilter.push({ relation_id: objectId }); - } - const truncate = await col - .find( - { - ...baseId, - connection_id: connection_id, - _id: { $ne: doc._id }, - $or: truncateFilter - }, - { session } - ) - .toArray(); - dropTables = truncate.map( - (doc) => - new storage.SourceTable({ - id: doc._id, - connectionTag: connection_tag, - objectId: doc.relation_id, - schema: doc.schema_name, - name: doc.table_name, - replicaIdColumns: - doc.replica_id_columns2?.map((c) => ({ name: c.name, typeOid: c.type_oid, type: c.type })) ?? [], - snapshotComplete: doc.snapshot_done ?? true - }) - ); - - result = { - table: sourceTable, - dropTables: dropTables - }; - }); - if (initializeSourceRecordsFor != null) { - await this.db.initializeSourceRecordsCollection(group_id, initializeSourceRecordsFor); - } - return result!; + resolveTable(options: storage.ResolveTableOptions): Promise { + return this.impl.resolveTable(options); } - async getParameterSets( - checkpoint: MongoReplicationCheckpoint, + getParameterSets( + checkpoint: storage.ReplicationCheckpoint & { snapshotTime: bson.Timestamp }, lookups: ScopedParameterLookup[] ): Promise { - if (this.db.storageConfig.incrementalReprocessing) { - return getParameterSetsV3(this.versionContext, checkpoint, lookups); - } - return getParameterSetsV1(this.versionContext, checkpoint, lookups); + return this.impl.getParameterSets(checkpoint as any, lookups); } - async *getBucketDataBatch( + getBucketDataBatch( checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions ): AsyncIterable { - if (this.db.storageConfig.incrementalReprocessing) { - yield* getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options); - return; - } - yield* getBucketDataBatchV1(this.versionContext, checkpoint, dataBuckets, options); - } - - async getChecksums( - checkpoint: utils.InternalOpId, - buckets: storage.BucketChecksumRequest[] - ): Promise { - return this.checksums.getChecksums(checkpoint, buckets); - } - - clearChecksumCache() { - this.checksums.clearCache(); - } - - async terminate(options?: storage.TerminateOptions) { - // Default is to clear the storage except when explicitly requested not to. - if (!options || options?.clearStorage) { - await this.clear(options); - } - await this.db.sync_rules.updateOne( - { - _id: this.group_id - }, - { - $set: { - state: storage.SyncRuleState.TERMINATED, - persisted_lsn: null, - snapshot_done: false - } - } - ); - await this.db.notifyCheckpoint(); - } - - async getStatus(): Promise { - const doc = await this.db.sync_rules.findOne( - { - _id: this.group_id - }, - { - projection: { - snapshot_done: 1, - last_checkpoint_lsn: 1, - state: 1, - snapshot_lsn: 1 - } - } - ); - if (doc == null) { - throw new ServiceAssertionError('Cannot find sync rules status'); - } - - return { - snapshot_done: doc.snapshot_done, - snapshot_lsn: doc.snapshot_lsn ?? null, - active: doc.state == 'ACTIVE', - checkpoint_lsn: doc.last_checkpoint_lsn - }; - } - - async clear(options?: storage.ClearStorageOptions): Promise { - const signal = options?.signal; - - if (signal?.aborted) { - throw new ReplicationAbortedError('Aborted clearing data', signal.reason); - } - - await this.db.sync_rules.updateOne( - { - _id: this.group_id - }, - { - $set: { - snapshot_done: false, - persisted_lsn: null, - last_checkpoint_lsn: null, - last_checkpoint: null, - no_checkpoint_before: null - }, - $unset: { - snapshot_lsn: 1 - } - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ); - if (this.db.storageConfig.incrementalReprocessing) { - for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { - await collection.drop(); - } - } else { - await this.clearDeleteMany( - 'bucket data', - () => - this.db.bucket_data.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['b', 'o']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ), - signal - ); - } - if (this.db.storageConfig.incrementalReprocessing) { - for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { - await collection.collection.drop(); - } - } else { - await this.clearDeleteMany( - 'parameter index', - () => - this.db.parameterIndexV1.deleteMany( - { - 'key.g': this.group_id - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ), - signal - ); - } - - for (const collection of await this.db.listSourceRecordCollectionsV3(this.group_id)) { - await collection.drop(); - } - - if (this.db.storageConfig.incrementalReprocessing) { - await this.db - .bucketStateV3(this.group_id) - .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) - .catch((error) => { - if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { - return; - } - throw error; - }); - } else { - await this.clearDeleteMany( - 'bucket state', - () => - this.db.bucketStateV1.deleteMany( - { - _id: idPrefixFilter({ g: this.group_id }, ['b']) - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ), - signal - ); - } - - if (this.db.storageConfig.incrementalReprocessing) { - await this.db - .sourceTablesV3(this.group_id) - .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) - .catch((error) => { - if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { - return; - } - throw error; - }); - } else { - await this.clearDeleteMany( - 'source tables', - () => - this.db.commonSourceTables(this.group_id).deleteMany( - { - group_id: this.group_id - }, - { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } - ), - signal - ); - } - - this.#storageInitialized = false; - } - - private async clearDeleteMany( - label: string, - operation: () => Promise, - signal?: AbortSignal - ): Promise { - await retryOnMongoMaxTimeMSExpired(operation, { - signal, - abortMessage: 'Aborted clearing data', - retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5, - onRetry: () => { - logger.info( - `${this.slot_name} Cleared batch of ${label} in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` - ); - } - }); - } - - async reportError(e: any): Promise { - const message = String(e.message ?? 'Replication failure'); - await this.db.sync_rules.updateOne( - { - _id: this.group_id - }, - { - $set: { - last_fatal_error: message, - last_fatal_error_ts: new Date() - } - } - ); - } - - async compact(options?: storage.CompactOptions) { - let maxOpId = options?.maxOpId; - if (maxOpId == null) { - const checkpoint = await this.getCheckpointInternal(); - maxOpId = checkpoint?.checkpoint ?? undefined; - } - await new MongoCompactor(this, this.db, { ...options, maxOpId }).compact(); - - if (maxOpId != null && options?.compactParameterData) { - await new MongoParameterCompactor(this.db, this.group_id, maxOpId, options).compact(); - } + return this.impl.getBucketDataBatch(checkpoint, dataBuckets, options); } - async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise { - logger.info(`Populating persistent checksum cache...`); - const start = Date.now(); - // We do a minimal compact here. - // We can optimize this in the future. - const compactor = new MongoCompactor(this, this.db, { - ...options, - // Don't track updates for MOVE compacting - memoryLimitMB: 0 - }); - - const result = await compactor.populateChecksums({ - // There are cases with millions of small buckets, in which case it can take very long to - // populate the checksums, with minimal benefit. We skip the small buckets here. - minBucketChanges: options.minBucketChanges ?? 10 - }); - const duration = Date.now() - start; - logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`); - return result; + getChecksums(checkpoint: utils.InternalOpId, buckets: storage.BucketChecksumRequest[]): Promise { + return this.impl.getChecksums(checkpoint, buckets); } - /** - * Instance-wide watch on the latest available checkpoint (op_id + lsn). - */ - private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable { - if (signal.aborted) { - return; - } - - // If the stream is idle, we wait a max of a minute (CHECKPOINT_TIMEOUT_MS) before we get another checkpoint, - // to avoid stale checkpoint snapshots. This is what checkpointTimeoutStream() is for. - // Essentially, even if there are no actual checkpoint changes, we want a new snapshotTime every minute or so, - // to ensure that any new clients connecting will get a valid snapshotTime. - const stream = mergeAsyncIterables( - [this.checkpointChangesStream(signal), this.checkpointTimeoutStream(signal)], - signal - ); - - // We only watch changes to the active sync rules. - // If it changes to inactive, we abort and restart with the new sync rules. - for await (const _ of stream) { - if (signal.aborted) { - // Would likely have been caught by the signal on the timeout or the upstream stream, but we check here anyway - break; - } - - const op = await this.getCheckpointInternal(); - if (op == null) { - // Sync rules have changed - abort and restart. - // We do a soft close of the stream here - no error - break; - } - - // Previously, we only yielded when the checkpoint or lsn changed. - // However, we always want to use the latest snapshotTime, so we skip that filtering here. - // That filtering could be added in the per-user streams if needed, but in general the capped collection - // should already only contain useful changes in most cases. - yield op; - } + clearChecksumCache(): void { + this.impl.clearChecksumCache(); } - // Nothing is done here until a subscriber starts to iterate - private readonly sharedIter = new BroadcastIterable((signal) => { - return this.watchActiveCheckpoint(signal); - }); - - /** - * User-specific watch on the latest checkpoint and/or write checkpoint. - */ - async *watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable { - let lastCheckpoint: ReplicationCheckpoint | null = null; - - const iter = this.sharedIter[Symbol.asyncIterator](options.signal); - - let writeCheckpoint: bigint | null = null; - // true if we queried the initial write checkpoint, even if it doesn't exist - let queriedInitialWriteCheckpoint = false; - - for await (const nextCheckpoint of iter) { - // lsn changes are not important by itself. - // What is important is: - // 1. checkpoint (op_id) changes. - // 2. write checkpoint changes for the specific user - - if (nextCheckpoint.lsn != null && !queriedInitialWriteCheckpoint) { - // Lookup the first write checkpoint for the user when we can. - // There will not actually be one in all cases. - writeCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({ - sync_rules_id: this.group_id, - user_id: options.user_id, - heads: { - '1': nextCheckpoint.lsn - } - }); - queriedInitialWriteCheckpoint = true; - } - - if ( - lastCheckpoint != null && - lastCheckpoint.checkpoint == nextCheckpoint.checkpoint && - lastCheckpoint.lsn == nextCheckpoint.lsn - ) { - // No change - wait for next one - // In some cases, many LSNs may be produced in a short time. - // Add a delay to throttle the loop a bit. - await timers.setTimeout(20 + 10 * Math.random()); - continue; - } - - if (lastCheckpoint == null) { - // First message for this stream - "INVALIDATE_ALL" means it will lookup all data - yield { - base: nextCheckpoint, - writeCheckpoint, - update: CHECKPOINT_INVALIDATE_ALL - }; - } else { - const updates = await this.getCheckpointChanges({ - lastCheckpoint, - nextCheckpoint - }); - - let updatedWriteCheckpoint = updates.updatedWriteCheckpoints.get(options.user_id) ?? null; - if (updates.invalidateWriteCheckpoints) { - // Invalidated means there were too many updates to track the individual ones, - // so we switch to "polling" (querying directly in each stream). - updatedWriteCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({ - sync_rules_id: this.group_id, - user_id: options.user_id, - heads: { - '1': nextCheckpoint.lsn! - } - }); - } - if (updatedWriteCheckpoint != null && (writeCheckpoint == null || updatedWriteCheckpoint > writeCheckpoint)) { - writeCheckpoint = updatedWriteCheckpoint; - // If it happened that we haven't queried a write checkpoint at this point, - // then we don't need to anymore, since we got an updated one. - queriedInitialWriteCheckpoint = true; - } - - yield { - base: nextCheckpoint, - writeCheckpoint, - update: { - updatedDataBuckets: updates.updatedDataBuckets, - invalidateDataBuckets: updates.invalidateDataBuckets, - updatedParameterLookups: updates.updatedParameterLookups, - invalidateParameterBuckets: updates.invalidateParameterBuckets - } - }; - } - - lastCheckpoint = nextCheckpoint; - } + terminate(options?: storage.TerminateOptions): Promise { + return this.impl.terminate(options); } - /** - * This watches the checkpoint_events capped collection for new documents inserted, - * and yields whenever one or more documents are inserted. - * - * The actual checkpoint must be queried on the sync_rules collection after this. - */ - private async *checkpointChangesStream(signal: AbortSignal): AsyncGenerator { - if (signal.aborted) { - return; - } - - const query = () => { - return this.db.checkpoint_events.find( - {}, - { tailable: true, awaitData: true, maxAwaitTimeMS: 10_000, batchSize: 1000 } - ); - }; - - let cursor = query(); - - signal.addEventListener('abort', () => { - cursor.close().catch(() => {}); - }); - - // Yield once on start, regardless of whether there are documents in the cursor. - // This is to ensure that the first iteration of the generator yields immediately. - yield; - - try { - while (!signal.aborted) { - const doc = await cursor.tryNext().catch((e) => { - if (lib_mongo.isMongoServerError(e) && e.codeName === 'CappedPositionLost') { - // Cursor position lost, potentially due to a high rate of notifications - cursor = query(); - // Treat as an event found, before querying the new cursor again - return {}; - } else { - return Promise.reject(e); - } - }); - if (cursor.closed) { - return; - } - // Skip buffered documents, if any. We don't care about the contents, - // we only want to know when new documents are inserted. - cursor.readBufferedDocuments(); - if (doc != null) { - yield; - } - } - } catch (e) { - if (signal.aborted) { - return; - } - throw e; - } finally { - await cursor.close(); - } + getStatus(): Promise { + return this.impl.getStatus(); } - private async *checkpointTimeoutStream(signal: AbortSignal): AsyncGenerator { - while (!signal.aborted) { - try { - await timers.setTimeout(CHECKPOINT_TIMEOUT_MS, undefined, { signal }); - } catch (e) { - if (e.name == 'AbortError') { - // This is how we typically abort this stream, when all listeners are done - return; - } - throw e; - } - - if (!signal.aborted) { - yield; - } - } + clear(options?: storage.ClearStorageOptions): Promise { + return this.impl.clear(options); } - private async getDataBucketChanges( - options: GetCheckpointChangesOptions - ): Promise> { - if (this.db.storageConfig.incrementalReprocessing) { - return getDataBucketChangesV3(this.versionContext, options); - } - return getDataBucketChangesV1(this.versionContext, options); + reportError(e: any): Promise { + return this.impl.reportError(e); } - private async getParameterBucketChanges( - options: GetCheckpointChangesOptions - ): Promise> { - if (this.db.storageConfig.incrementalReprocessing) { - return getParameterBucketChangesV3(this.versionContext, options); - } - return getParameterBucketChangesV1(this.versionContext, options); + compact(options?: storage.CompactOptions): Promise { + return this.impl.compact(options); } - // If we processed all connections together for each checkpoint, we could do a single lookup for all connections. - // In practice, specific connections may fall behind. So instead, we just cache the results of each specific lookup. - // TODO (later): - // We can optimize this by implementing it like ChecksumCache: We can use partial cache results to do - // more efficient lookups in some cases. - private checkpointChangesCache = new LRUCache< - string, - InternalCheckpointChanges, - { options: GetCheckpointChangesOptions } - >({ - // Limit to 50 cache entries, or 10MB, whichever comes first. - // Some rough calculations: - // If we process 10 checkpoints per second, and a connection may be 2 seconds behind, we could have - // up to 20 relevant checkpoints. That gives us 20*20 = 400 potentially-relevant cache entries. - // That is a worst-case scenario, so we don't actually store that many. In real life, the cache keys - // would likely be clustered around a few values, rather than spread over all 400 potential values. - max: 50, - maxSize: 12 * 1024 * 1024, - sizeCalculation: (value: InternalCheckpointChanges) => { - // Estimate of memory usage - const paramSize = [...value.updatedParameterLookups].reduce((a, b) => a + b.length, 0); - const bucketSize = [...value.updatedDataBuckets].reduce((a, b) => a + b.length, 0); - const writeCheckpointSize = value.updatedWriteCheckpoints.size * 30; // estiamte for user_id + bigint - return 100 + paramSize + bucketSize + writeCheckpointSize; - }, - fetchMethod: async (_key, _staleValue, options) => { - return this.getCheckpointChangesInternal(options.context.options); - } - }); - - async getCheckpointChanges(options: GetCheckpointChangesOptions): Promise { - const key = `${options.lastCheckpoint.checkpoint}_${options.lastCheckpoint.lsn}__${options.nextCheckpoint.checkpoint}_${options.nextCheckpoint.lsn}`; - const result = await this.checkpointChangesCache.fetch(key, { context: { options } }); - return result!; + populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise { + return this.impl.populatePersistentChecksumCache(options); } - private async getCheckpointChangesInternal(options: GetCheckpointChangesOptions): Promise { - const dataUpdates = await this.getDataBucketChanges(options); - const parameterUpdates = await this.getParameterBucketChanges(options); - const writeCheckpointUpdates = await this.writeCheckpointAPI.getWriteCheckpointChanges(options); - - return { - ...dataUpdates, - ...parameterUpdates, - ...writeCheckpointUpdates - }; + watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable { + return this.impl.watchCheckpointChanges(options); } -} - -interface InternalCheckpointChanges extends CheckpointChanges { - updatedWriteCheckpoints: Map; - invalidateWriteCheckpoints: boolean; -} - -class MongoReplicationCheckpoint implements ReplicationCheckpoint { - constructor( - private storage: MongoSyncBucketStorage, - public readonly checkpoint: InternalOpId, - public readonly lsn: string | null, - public snapshotTime: mongo.Timestamp - ) {} - - async getParameterSets(lookups: ScopedParameterLookup[]): Promise { - return this.storage.getParameterSets(this, lookups); - } -} - -class EmptyReplicationCheckpoint implements ReplicationCheckpoint { - readonly checkpoint: InternalOpId = 0n; - readonly lsn: string | null = null; - async getParameterSets(lookups: ScopedParameterLookup[]): Promise { - return []; + getCheckpointChanges(options: GetCheckpointChangesOptions): Promise { + return this.impl.getCheckpointChanges(options); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts new file mode 100644 index 000000000..e7fbd8865 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts @@ -0,0 +1,784 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { + BaseObserver, + logger, + ReplicationAbortedError, + ServiceAssertionError +} from '@powersync/lib-services-framework'; +import { + BroadcastIterable, + CHECKPOINT_INVALIDATE_ALL, + CheckpointChanges, + GetCheckpointChangesOptions, + InternalOpId, + maxLsn, + mergeAsyncIterables, + PopulateChecksumCacheOptions, + PopulateChecksumCacheResults, + ReplicationCheckpoint, + storage, + utils, + WatchWriteCheckpointOptions +} from '@powersync/service-core'; +import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; +import { LRUCache } from 'lru-cache'; +import * as timers from 'timers/promises'; +import { idPrefixFilter, retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; +import { MongoBucketStorage } from '../../MongoBucketStorage.js'; +import { VersionedPowerSyncMongo } from '../db.js'; +import { + BucketDataKeyV1, + BucketStateDocument, + CommonSourceTableDocument, + SourceKey, + StorageConfig +} from '../models.js'; +import { MongoChecksumOptions, MongoChecksums } from '../MongoChecksums.js'; +import { MongoCompactor } from '../MongoCompactor.js'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; +import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; +import { MongoWriteCheckpointAPI } from '../MongoWriteCheckpointAPI.js'; +import { MongoSyncBucketStorageContext } from './MongoSyncBucketStorageContext.js'; +import { MongoBucketBatchOptions } from './MongoBucketBatch.js'; + +export interface MongoSyncBucketStorageOptions { + checksumOptions?: Omit; + storageConfig: StorageConfig; +} + +interface InternalCheckpointChanges extends CheckpointChanges { + updatedWriteCheckpoints: Map; + invalidateWriteCheckpoints: boolean; +} + +/** + * Only keep checkpoints around for a minute, before fetching a fresh one. + * + * The reason is that we keep a MongoDB snapshot reference (clusterTime) with the checkpoint, + * and they expire after 5 minutes by default. This is an issue if the checkpoint stream is idle, + * but new clients connect and use an outdated checkpoint snapshot for parameter queries. + * + * These will be filtered out for existing clients, so should not create significant overhead. + */ +const CHECKPOINT_TIMEOUT_MS = 60_000; + +export abstract class BaseMongoSyncBucketStorage + extends BaseObserver + implements storage.SyncRulesBucketStorage +{ + readonly db: VersionedPowerSyncMongo; + readonly checksums: MongoChecksums; + + private parsedSyncRulesCache: { parsed: HydratedSyncRules; options: storage.ParseSyncRulesOptions } | undefined; + private writeCheckpointAPI: MongoWriteCheckpointAPI; + #storageInitialized = false; + + constructor( + public readonly factory: MongoBucketStorage, + public readonly group_id: number, + protected readonly sync_rules: MongoPersistedSyncRulesContent, + public readonly slot_name: string, + writeCheckpointMode: storage.WriteCheckpointMode | undefined, + options: MongoSyncBucketStorageOptions + ) { + super(); + this.db = factory.db.versioned(sync_rules.getStorageConfig()); + this.checksums = new MongoChecksums(this.db, this.group_id, { + ...options.checksumOptions, + storageConfig: options?.storageConfig, + mapping: sync_rules.mapping + }); + this.writeCheckpointAPI = new MongoWriteCheckpointAPI({ + db: this.db, + mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED, + sync_rules_id: group_id + }); + } + + get writeCheckpointMode() { + return this.writeCheckpointAPI.writeCheckpointMode; + } + + get mapping() { + return this.sync_rules.mapping; + } + + protected get versionContext(): MongoSyncBucketStorageContext { + return { + db: this.db, + group_id: this.group_id, + mapping: this.mapping + }; + } + + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { + this.writeCheckpointAPI.setWriteCheckpointMode(mode); + } + + createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { + return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint); + } + + lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { + return this.writeCheckpointAPI.lastWriteCheckpoint({ + ...filters, + sync_rules_id: this.group_id + }); + } + + getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncRules { + const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; + if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { + this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncRules(), options }; + } + + return this.parsedSyncRulesCache!.parsed; + } + + async getCheckpoint(): Promise { + return (await this.getCheckpointInternal()) ?? new EmptyReplicationCheckpoint(); + } + + async getCheckpointInternal(): Promise { + return await this.db.client.withSession({ snapshot: true }, async (session) => { + const doc = await this.db.sync_rules.findOne( + { _id: this.group_id }, + { + session, + projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 } + } + ); + if (!doc?.snapshot_done || !['ACTIVE', 'ERRORED'].includes(doc.state)) { + return null; + } + + const snapshotTime = (session as any).snapshotTime as bson.Timestamp | undefined; + if (snapshotTime == null) { + throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()'); + } + return new MongoReplicationCheckpoint( + this, + doc.last_checkpoint ?? 0n, + doc.last_checkpoint_lsn ?? null, + snapshotTime + ); + }); + } + + protected abstract initializeVersionStorage(): Promise; + + private async initializeStorage() { + if (this.#storageInitialized) { + return; + } + + await this.db.initializeStreamStorage(this.group_id); + await this.initializeVersionStorage(); + this.#storageInitialized = true; + } + + protected abstract createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch; + + async createWriter(options: storage.CreateWriterOptions): Promise { + await this.initializeStorage(); + + const doc = await this.db.sync_rules.findOne( + { + _id: this.group_id + }, + { projection: { last_checkpoint_lsn: 1, no_checkpoint_before: 1, keepalive_op: 1, snapshot_lsn: 1 } } + ); + const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null; + + const batchOptions = { + logger: options.logger, + db: this.db, + syncRules: this.sync_rules.parsed(options).hydratedSyncRules(), + mapping: this.sync_rules.mapping, + groupId: this.group_id, + slotName: this.slot_name, + lastCheckpointLsn: checkpoint_lsn, + resumeFromLsn: maxLsn(checkpoint_lsn, doc?.snapshot_lsn), + keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null, + storeCurrentData: options.storeCurrentData, + skipExistingRows: options.skipExistingRows ?? false, + markRecordUnavailable: options.markRecordUnavailable + }; + const writer = this.createWriterImpl(batchOptions); + this.iterateListeners((cb) => cb.batchStarted?.(writer)); + return writer; + } + + async startBatch( + options: storage.CreateWriterOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + await using writer = await this.createWriter(options); + await callback(writer); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + } + + protected abstract sourceTableBaseId(): Partial; + + protected abstract augmentCreatedSourceTableDocument( + createDoc: CommonSourceTableDocument, + options: storage.ResolveTableOptions, + candidateSourceTable: storage.SourceTable + ): void; + + protected abstract initializeResolvedSourceRecords(sourceTableId: bson.ObjectId): Promise; + + async resolveTable(options: storage.ResolveTableOptions): Promise { + const { group_id, connection_id, connection_tag, entity_descriptor } = options; + + const { schema, name, objectId, replicaIdColumns } = entity_descriptor; + + const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({ + name: column.name, + type: column.type, + type_oid: column.typeId + })); + let result: storage.ResolveTableResult | null = null; + let initializeSourceRecordsFor: bson.ObjectId | null = null; + + const baseId = this.sourceTableBaseId(); + await this.db.client.withSession(async (session) => { + const col = this.db.commonSourceTables(group_id); + let filter: Partial = { + ...baseId, + connection_id: connection_id, + schema_name: schema, + table_name: name, + replica_id_columns2: normalizedReplicaIdColumns + }; + + if (objectId != null) { + filter.relation_id = objectId; + } + let doc = await col.findOne(filter, { session }); + if (doc == null) { + const candidateSourceTable = new storage.SourceTable({ + id: new bson.ObjectId(), + connectionTag: connection_tag, + objectId: objectId, + schema: schema, + name: name, + replicaIdColumns: replicaIdColumns, + snapshotComplete: false + }); + const createDoc: CommonSourceTableDocument = { + _id: candidateSourceTable.id as bson.ObjectId, + ...(baseId as any), + connection_id: connection_id, + relation_id: objectId, + schema_name: schema, + table_name: name, + replica_id_columns: null, + replica_id_columns2: normalizedReplicaIdColumns, + snapshot_done: false, + snapshot_status: undefined + }; + this.augmentCreatedSourceTableDocument(createDoc, options, candidateSourceTable); + doc = createDoc; + + await col.insertOne(doc, { session }); + initializeSourceRecordsFor = doc._id; + } + const sourceTable = new storage.SourceTable({ + id: doc._id, + connectionTag: connection_tag, + objectId: objectId, + schema: schema, + name: name, + replicaIdColumns: replicaIdColumns, + snapshotComplete: doc.snapshot_done ?? true + }); + sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); + sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); + sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable); + sourceTable.snapshotStatus = + doc.snapshot_status == null + ? undefined + : { + lastKey: doc.snapshot_status.last_key?.buffer ?? null, + totalEstimatedCount: doc.snapshot_status.total_estimated_count, + replicatedCount: doc.snapshot_status.replicated_count + }; + + let dropTables: storage.SourceTable[] = []; + let truncateFilter = [{ schema_name: schema, table_name: name }] as any[]; + if (objectId != null) { + truncateFilter.push({ relation_id: objectId }); + } + const truncate = await col + .find( + { + ...baseId, + connection_id: connection_id, + _id: { $ne: doc._id }, + $or: truncateFilter + }, + { session } + ) + .toArray(); + dropTables = truncate.map( + (doc) => + new storage.SourceTable({ + id: doc._id, + connectionTag: connection_tag, + objectId: doc.relation_id, + schema: doc.schema_name, + name: doc.table_name, + replicaIdColumns: + doc.replica_id_columns2?.map((c) => ({ name: c.name, typeOid: c.type_oid, type: c.type })) ?? [], + snapshotComplete: doc.snapshot_done ?? true + }) + ); + + result = { + table: sourceTable, + dropTables: dropTables + }; + }); + if (initializeSourceRecordsFor != null) { + await this.initializeResolvedSourceRecords(initializeSourceRecordsFor); + } + return result!; + } + + protected abstract getParameterSetsImpl( + checkpoint: MongoReplicationCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise; + + async getParameterSets( + checkpoint: MongoReplicationCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise { + return this.getParameterSetsImpl(checkpoint, lookups); + } + + protected abstract getBucketDataBatchImpl( + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions + ): AsyncIterable; + + async *getBucketDataBatch( + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions + ): AsyncIterable { + yield* this.getBucketDataBatchImpl(checkpoint, dataBuckets, options); + } + + async getChecksums( + checkpoint: utils.InternalOpId, + buckets: storage.BucketChecksumRequest[] + ): Promise { + return this.checksums.getChecksums(checkpoint, buckets); + } + + clearChecksumCache() { + this.checksums.clearCache(); + } + + async terminate(options?: storage.TerminateOptions) { + if (!options || options?.clearStorage) { + await this.clear(options); + } + await this.db.sync_rules.updateOne( + { + _id: this.group_id + }, + { + $set: { + state: storage.SyncRuleState.TERMINATED, + persisted_lsn: null, + snapshot_done: false + } + } + ); + await this.db.notifyCheckpoint(); + } + + async getStatus(): Promise { + const doc = await this.db.sync_rules.findOne( + { + _id: this.group_id + }, + { + projection: { + snapshot_done: 1, + last_checkpoint_lsn: 1, + state: 1, + snapshot_lsn: 1 + } + } + ); + if (doc == null) { + throw new ServiceAssertionError('Cannot find sync rules status'); + } + + return { + snapshot_done: doc.snapshot_done, + snapshot_lsn: doc.snapshot_lsn ?? null, + active: doc.state == 'ACTIVE', + checkpoint_lsn: doc.last_checkpoint_lsn + }; + } + + protected abstract clearBucketData(signal?: AbortSignal): Promise; + + protected abstract clearParameterIndexes(signal?: AbortSignal): Promise; + + protected abstract clearBucketState(signal?: AbortSignal): Promise; + + protected abstract clearSourceTables(signal?: AbortSignal): Promise; + + async clear(options?: storage.ClearStorageOptions): Promise { + const signal = options?.signal; + + if (signal?.aborted) { + throw new ReplicationAbortedError('Aborted clearing data', signal.reason); + } + + await this.db.sync_rules.updateOne( + { + _id: this.group_id + }, + { + $set: { + snapshot_done: false, + persisted_lsn: null, + last_checkpoint_lsn: null, + last_checkpoint: null, + no_checkpoint_before: null + }, + $unset: { + snapshot_lsn: 1 + } + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ); + + await this.clearBucketData(signal); + await this.clearParameterIndexes(signal); + + for (const collection of await this.db.listSourceRecordCollectionsV3(this.group_id)) { + await collection.drop(); + } + + await this.clearBucketState(signal); + await this.clearSourceTables(signal); + + this.#storageInitialized = false; + } + + protected async clearDeleteMany( + label: string, + operation: () => Promise, + signal?: AbortSignal + ): Promise { + await retryOnMongoMaxTimeMSExpired(operation, { + signal, + abortMessage: 'Aborted clearing data', + retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5, + onRetry: () => { + logger.info( + `${this.slot_name} Cleared batch of ${label} in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` + ); + } + }); + } + + async reportError(e: any): Promise { + const message = String(e.message ?? 'Replication failure'); + await this.db.sync_rules.updateOne( + { + _id: this.group_id + }, + { + $set: { + last_fatal_error: message, + last_fatal_error_ts: new Date() + } + } + ); + } + + async compact(options?: storage.CompactOptions) { + let maxOpId = options?.maxOpId; + if (maxOpId == null) { + const checkpoint = await this.getCheckpointInternal(); + maxOpId = checkpoint?.checkpoint ?? undefined; + } + await new MongoCompactor(this as any, this.db, { ...options, maxOpId }).compact(); + + if (maxOpId != null && options?.compactParameterData) { + await new MongoParameterCompactor(this.db, this.group_id, maxOpId, options).compact(); + } + } + + async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise { + logger.info(`Populating persistent checksum cache...`); + const start = Date.now(); + const compactor = new MongoCompactor(this as any, this.db, { + ...options, + memoryLimitMB: 0 + }); + + const result = await compactor.populateChecksums({ + minBucketChanges: options.minBucketChanges ?? 10 + }); + const duration = Date.now() - start; + logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`); + return result; + } + + private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable { + if (signal.aborted) { + return; + } + + const stream = mergeAsyncIterables( + [this.checkpointChangesStream(signal), this.checkpointTimeoutStream(signal)], + signal + ); + + for await (const _ of stream) { + if (signal.aborted) { + break; + } + + const op = await this.getCheckpointInternal(); + if (op == null) { + break; + } + + yield op; + } + } + + private readonly sharedIter = new BroadcastIterable((signal) => { + return this.watchActiveCheckpoint(signal); + }); + + async *watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable { + let lastCheckpoint: ReplicationCheckpoint | null = null; + + const iter = this.sharedIter[Symbol.asyncIterator](options.signal); + + let writeCheckpoint: bigint | null = null; + let queriedInitialWriteCheckpoint = false; + + for await (const nextCheckpoint of iter) { + if (nextCheckpoint.lsn != null && !queriedInitialWriteCheckpoint) { + writeCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({ + sync_rules_id: this.group_id, + user_id: options.user_id, + heads: { + '1': nextCheckpoint.lsn + } + }); + queriedInitialWriteCheckpoint = true; + } + + if ( + lastCheckpoint != null && + lastCheckpoint.checkpoint == nextCheckpoint.checkpoint && + lastCheckpoint.lsn == nextCheckpoint.lsn + ) { + await timers.setTimeout(20 + 10 * Math.random()); + continue; + } + + if (lastCheckpoint == null) { + yield { + base: nextCheckpoint, + writeCheckpoint, + update: CHECKPOINT_INVALIDATE_ALL + }; + } else { + const updates = await this.getCheckpointChanges({ + lastCheckpoint, + nextCheckpoint + }); + + let updatedWriteCheckpoint = updates.updatedWriteCheckpoints.get(options.user_id) ?? null; + if (updates.invalidateWriteCheckpoints) { + updatedWriteCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({ + sync_rules_id: this.group_id, + user_id: options.user_id, + heads: { + '1': nextCheckpoint.lsn! + } + }); + } + if (updatedWriteCheckpoint != null && (writeCheckpoint == null || updatedWriteCheckpoint > writeCheckpoint)) { + writeCheckpoint = updatedWriteCheckpoint; + queriedInitialWriteCheckpoint = true; + } + + yield { + base: nextCheckpoint, + writeCheckpoint, + update: { + updatedDataBuckets: updates.updatedDataBuckets, + invalidateDataBuckets: updates.invalidateDataBuckets, + updatedParameterLookups: updates.updatedParameterLookups, + invalidateParameterBuckets: updates.invalidateParameterBuckets + } + }; + } + + lastCheckpoint = nextCheckpoint; + } + } + + private async *checkpointChangesStream(signal: AbortSignal): AsyncGenerator { + if (signal.aborted) { + return; + } + + const query = () => { + return this.db.checkpoint_events.find( + {}, + { tailable: true, awaitData: true, maxAwaitTimeMS: 10_000, batchSize: 1000 } + ); + }; + + let cursor = query(); + + signal.addEventListener('abort', () => { + cursor.close().catch(() => {}); + }); + + yield; + + try { + while (!signal.aborted) { + const doc = await cursor.tryNext().catch((e) => { + if (lib_mongo.isMongoServerError(e) && e.codeName === 'CappedPositionLost') { + cursor = query(); + return {}; + } else { + return Promise.reject(e); + } + }); + if (cursor.closed) { + return; + } + cursor.readBufferedDocuments(); + if (doc != null) { + yield; + } + } + } catch (e) { + if (signal.aborted) { + return; + } + throw e; + } finally { + await cursor.close(); + } + } + + private async *checkpointTimeoutStream(signal: AbortSignal): AsyncGenerator { + while (!signal.aborted) { + try { + await timers.setTimeout(CHECKPOINT_TIMEOUT_MS, undefined, { signal }); + } catch (e) { + if (e.name == 'AbortError') { + return; + } + throw e; + } + + if (!signal.aborted) { + yield; + } + } + } + + protected abstract getDataBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise>; + + private async getDataBucketChanges( + options: GetCheckpointChangesOptions + ): Promise> { + return this.getDataBucketChangesImpl(options); + } + + protected abstract getParameterBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise>; + + private async getParameterBucketChanges( + options: GetCheckpointChangesOptions + ): Promise> { + return this.getParameterBucketChangesImpl(options); + } + + private checkpointChangesCache = new LRUCache< + string, + InternalCheckpointChanges, + { options: GetCheckpointChangesOptions } + >({ + max: 50, + maxSize: 12 * 1024 * 1024, + sizeCalculation: (value: InternalCheckpointChanges) => { + const paramSize = [...value.updatedParameterLookups].reduce((a, b) => a + b.length, 0); + const bucketSize = [...value.updatedDataBuckets].reduce((a, b) => a + b.length, 0); + const writeCheckpointSize = value.updatedWriteCheckpoints.size * 30; + return 100 + paramSize + bucketSize + writeCheckpointSize; + }, + fetchMethod: async (_key, _staleValue, options) => { + return this.getCheckpointChangesInternal(options.context.options); + } + }); + + async getCheckpointChanges(options: GetCheckpointChangesOptions): Promise { + const key = `${options.lastCheckpoint.checkpoint}_${options.lastCheckpoint.lsn}__${options.nextCheckpoint.checkpoint}_${options.nextCheckpoint.lsn}`; + const result = await this.checkpointChangesCache.fetch(key, { context: { options } }); + return result!; + } + + private async getCheckpointChangesInternal(options: GetCheckpointChangesOptions): Promise { + const dataUpdates = await this.getDataBucketChanges(options); + const parameterUpdates = await this.getParameterBucketChanges(options); + const writeCheckpointUpdates = await this.writeCheckpointAPI.getWriteCheckpointChanges(options); + + return { + ...dataUpdates, + ...parameterUpdates, + ...writeCheckpointUpdates + }; + } +} + +class MongoReplicationCheckpoint implements ReplicationCheckpoint { + constructor( + private storage: BaseMongoSyncBucketStorage, + public readonly checkpoint: InternalOpId, + public readonly lsn: string | null, + public snapshotTime: mongo.Timestamp + ) {} + + async getParameterSets(lookups: ScopedParameterLookup[]): Promise { + return this.storage.getParameterSets(this, lookups); + } +} + +class EmptyReplicationCheckpoint implements ReplicationCheckpoint { + readonly checkpoint: InternalOpId = 0n; + readonly lsn: string | null = null; + + async getParameterSets(_lookups: ScopedParameterLookup[]): Promise { + return []; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index b15fa5f32..66703552a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -13,12 +13,138 @@ import { import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { JSONBig } from '@powersync/service-jsonbig'; -import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; -import { BucketDataDocumentV1, LEGACY_BUCKET_DATA_DEFINITION_ID, bucketDataDocumentToTagged } from '../models.js'; +import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; +import { + BucketDataDocumentV1, + BucketDataKeyV1, + BucketStateDocument, + CommonSourceTableDocument, + LEGACY_BUCKET_DATA_DEFINITION_ID, + bucketDataDocumentToTagged +} from '../models.js'; import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js'; +import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorageBase.js'; +import { MongoBucketStorage } from '../../MongoBucketStorage.js'; +import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; +import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; +import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; + +export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { + constructor( + factory: MongoBucketStorage, + group_id: number, + sync_rules: MongoPersistedSyncRulesContent, + slot_name: string, + writeCheckpointMode: storage.WriteCheckpointMode | undefined, + options: MongoSyncBucketStorageOptions + ) { + super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); + } + + protected async initializeVersionStorage(): Promise {} + + protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch { + return new MongoBucketBatchV1(batchOptions); + } + + protected sourceTableBaseId(): Partial { + return { group_id: this.group_id }; + } + + protected augmentCreatedSourceTableDocument( + _createDoc: CommonSourceTableDocument, + _options: storage.ResolveTableOptions, + _candidateSourceTable: storage.SourceTable + ): void {} + + protected async initializeResolvedSourceRecords(_sourceTableId: bson.ObjectId): Promise {} + + protected getParameterSetsImpl( + checkpoint: MongoSyncBucketStorageCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise { + return getParameterSetsV1(this.versionContext, checkpoint, lookups); + } + + protected getBucketDataBatchImpl( + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions + ): AsyncIterable { + return getBucketDataBatchV1(this.versionContext, checkpoint, dataBuckets, options); + } + + protected async clearBucketData(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'bucket data', + () => + this.db.bucket_data.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['b', 'o']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } + + protected async clearParameterIndexes(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'parameter index', + () => + this.db.parameterIndexV1.deleteMany( + { + 'key.g': this.group_id + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } + + protected async clearBucketState(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'bucket state', + () => + this.db.bucketStateV1.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['b']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } + + protected async clearSourceTables(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'source tables', + () => + this.db.commonSourceTables(this.group_id).deleteMany( + { + group_id: this.group_id + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } + + protected getDataBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise> { + return getDataBucketChangesV1(this.versionContext, options); + } + + protected getParameterBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise> { + return getParameterBucketChangesV1(this.versionContext, options); + } +} export async function getParameterSetsV1( ctx: MongoSyncBucketStorageContext, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 1a50a6766..946cda41e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -19,6 +19,145 @@ import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js'; +import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorageBase.js'; +import { MongoBucketStorage } from '../../MongoBucketStorage.js'; +import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; +import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; +import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; +import { CommonSourceTableDocument } from '../models.js'; + +export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { + constructor( + factory: MongoBucketStorage, + group_id: number, + sync_rules: MongoPersistedSyncRulesContent, + slot_name: string, + writeCheckpointMode: storage.WriteCheckpointMode | undefined, + options: MongoSyncBucketStorageOptions + ) { + super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); + } + + protected async initializeVersionStorage(): Promise { + const mapping = this.mapping; + for (let source of mapping.allBucketDefinitionIds()) { + const collection = this.db.bucket_data_v3(this.group_id, source).collectionName; + await this.db.db + .createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') { + return; + } + throw error; + }); + } + for (let indexId of mapping.allParameterIndexIds()) { + await this.db.parameterIndexV3(this.group_id, indexId).createIndex( + { + lookup: 1, + key: 1, + _id: -1 + }, + { + name: 'lookup_op_id' + } + ); + } + } + + protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch { + return new MongoBucketBatchV3(batchOptions); + } + + protected sourceTableBaseId(): Partial { + return {}; + } + + protected augmentCreatedSourceTableDocument( + createDoc: CommonSourceTableDocument, + options: storage.ResolveTableOptions, + candidateSourceTable: storage.SourceTable + ): void { + const bucketDataSourceIds = options.sync_rules.definition.bucketDataSources + .filter((source) => source.tableSyncsData(candidateSourceTable)) + .map((source) => this.mapping.bucketSourceId(source)); + const parameterLookupSourceIds = options.sync_rules.definition.bucketParameterLookupSources + .filter((source) => source.tableSyncsParameters(candidateSourceTable)) + .map((source) => this.mapping.parameterLookupId(source)); + + Object.assign(createDoc, { + bucket_data_source_ids: bucketDataSourceIds, + parameter_lookup_source_ids: parameterLookupSourceIds + }); + } + + protected async initializeResolvedSourceRecords(sourceTableId: bson.ObjectId): Promise { + await this.db.initializeSourceRecordsCollection(this.group_id, sourceTableId); + } + + protected getParameterSetsImpl( + checkpoint: MongoSyncBucketStorageCheckpoint, + lookups: ScopedParameterLookup[] + ): Promise { + return getParameterSetsV3(this.versionContext, checkpoint, lookups); + } + + protected getBucketDataBatchImpl( + checkpoint: utils.InternalOpId, + dataBuckets: storage.BucketDataRequest[], + options?: storage.BucketDataBatchOptions + ): AsyncIterable { + return getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options); + } + + protected async clearBucketData(_signal?: AbortSignal): Promise { + for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { + await collection.drop(); + } + } + + protected async clearParameterIndexes(_signal?: AbortSignal): Promise { + for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) { + await collection.collection.drop(); + } + } + + protected async clearBucketState(_signal?: AbortSignal): Promise { + await this.db + .bucketStateV3(this.group_id) + .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } + + protected async clearSourceTables(_signal?: AbortSignal): Promise { + await this.db + .sourceTablesV3(this.group_id) + .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }) + .catch((error) => { + if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') { + return; + } + throw error; + }); + } + + protected getDataBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise> { + return getDataBucketChangesV3(this.versionContext, options); + } + + protected getParameterBucketChangesImpl( + options: GetCheckpointChangesOptions + ): Promise> { + return getParameterBucketChangesV3(this.versionContext, options); + } +} export async function getParameterSetsV3( ctx: MongoSyncBucketStorageContext, From f999fe922fc0232478eae1f0eda9b9fbbd2a68aa Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 17:21:20 +0200 Subject: [PATCH 60/93] Split db implementations. --- .../src/storage/MongoBucketStorage.ts | 5 +- .../implementation/MongoSyncBucketStorage.ts | 154 +---------- .../implementation/common/MongoBucketBatch.ts | 2 +- .../implementation/common/MongoChecksums.ts | 2 +- .../common/MongoChecksumsBase.ts | 2 +- .../implementation/common/MongoCompactor.ts | 4 +- .../common/MongoCompactorBase.ts | 4 +- .../common/MongoParameterCompactor.ts | 2 +- .../common/MongoParameterCompactorBase.ts | 2 +- .../common/MongoSyncBucketStorageBase.ts | 2 +- .../common/MongoSyncBucketStorageContext.ts | 2 +- .../implementation/common/PersistedBatch.ts | 2 +- .../common/VersionedPowerSyncMongoBase.ts | 136 ++++++++++ .../src/storage/implementation/db.ts | 255 +----------------- .../implementation/v1/SourceRecordStoreV1.ts | 2 +- .../v1/VersionedPowerSyncMongoV1.ts | 101 +++++++ .../implementation/v3/MongoChecksumsV3.ts | 2 +- .../implementation/v3/SourceRecordStoreV3.ts | 2 +- .../v3/VersionedPowerSyncMongoV3.ts | 146 ++++++++++ 19 files changed, 423 insertions(+), 404 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 233ae65f6..8b8667b4e 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -10,7 +10,8 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { PowerSyncMongo } from './implementation/db.js'; import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js'; import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js'; -import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; +import { createMongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; +import type { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; import { generateSlotName } from '../utils/util.js'; import { MongoChecksumOptions } from './implementation/MongoChecksums.js'; import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; @@ -53,7 +54,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { id = Number(id); } const storageConfig = (syncRules as MongoPersistedSyncRulesContent).getStorageConfig(); - const storage = new MongoSyncBucketStorage( + const storage = createMongoSyncBucketStorage( this, id, syncRules as MongoPersistedSyncRulesContent, diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index dee84a6d8..2062c5ed6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -9,8 +9,6 @@ import { } from '@powersync/service-core'; import * as bson from 'bson'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; -import { VersionedPowerSyncMongo } from './db.js'; -import { MongoChecksums } from './MongoChecksums.js'; import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorageBase.js'; import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js'; @@ -18,147 +16,19 @@ import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js'; export { MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorageBase.js'; -export class MongoSyncBucketStorage implements storage.SyncRulesBucketStorage { - private readonly impl: BaseMongoSyncBucketStorage; +export type MongoSyncBucketStorage = BaseMongoSyncBucketStorage; - constructor( - factory: MongoBucketStorage, - group_id: number, - sync_rules: MongoPersistedSyncRulesContent, - slot_name: string, - writeCheckpointMode: storage.WriteCheckpointMode | undefined, - options: MongoSyncBucketStorageOptions - ) { - if (sync_rules.getStorageConfig().incrementalReprocessing) { - this.impl = new MongoSyncBucketStorageV3(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); - } else { - this.impl = new MongoSyncBucketStorageV1(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); - } +export function createMongoSyncBucketStorage( + factory: MongoBucketStorage, + group_id: number, + sync_rules: MongoPersistedSyncRulesContent, + slot_name: string, + writeCheckpointMode: storage.WriteCheckpointMode | undefined, + options: MongoSyncBucketStorageOptions +): MongoSyncBucketStorage { + if (sync_rules.getStorageConfig().incrementalReprocessing) { + return new MongoSyncBucketStorageV3(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); } - get factory(): MongoBucketStorage { - return this.impl.factory; - } - - get group_id(): number { - return this.impl.group_id; - } - - get slot_name(): string { - return this.impl.slot_name; - } - - get db(): VersionedPowerSyncMongo { - return this.impl.db; - } - - get checksums(): MongoChecksums { - return this.impl.checksums; - } - - get mapping() { - return this.impl.mapping; - } - - get writeCheckpointMode() { - return this.impl.writeCheckpointMode; - } - - registerListener(listener: Partial): () => void { - return this.impl.registerListener(listener); - } - - setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { - this.impl.setWriteCheckpointMode(mode); - } - - createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { - return this.impl.createManagedWriteCheckpoint(checkpoint); - } - - lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { - return this.impl.lastWriteCheckpoint(filters); - } - - getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncRules { - return this.impl.getParsedSyncRules(options); - } - - getCheckpoint(): Promise { - return this.impl.getCheckpoint(); - } - - getCheckpointInternal(): Promise { - return this.impl.getCheckpointInternal(); - } - - createWriter(options: storage.CreateWriterOptions): Promise { - return this.impl.createWriter(options); - } - - startBatch( - options: storage.CreateWriterOptions, - callback: (batch: storage.BucketStorageBatch) => Promise - ): Promise { - return this.impl.startBatch(options, callback); - } - - resolveTable(options: storage.ResolveTableOptions): Promise { - return this.impl.resolveTable(options); - } - - getParameterSets( - checkpoint: storage.ReplicationCheckpoint & { snapshotTime: bson.Timestamp }, - lookups: ScopedParameterLookup[] - ): Promise { - return this.impl.getParameterSets(checkpoint as any, lookups); - } - - getBucketDataBatch( - checkpoint: utils.InternalOpId, - dataBuckets: storage.BucketDataRequest[], - options?: storage.BucketDataBatchOptions - ): AsyncIterable { - return this.impl.getBucketDataBatch(checkpoint, dataBuckets, options); - } - - getChecksums(checkpoint: utils.InternalOpId, buckets: storage.BucketChecksumRequest[]): Promise { - return this.impl.getChecksums(checkpoint, buckets); - } - - clearChecksumCache(): void { - this.impl.clearChecksumCache(); - } - - terminate(options?: storage.TerminateOptions): Promise { - return this.impl.terminate(options); - } - - getStatus(): Promise { - return this.impl.getStatus(); - } - - clear(options?: storage.ClearStorageOptions): Promise { - return this.impl.clear(options); - } - - reportError(e: any): Promise { - return this.impl.reportError(e); - } - - compact(options?: storage.CompactOptions): Promise { - return this.impl.compact(options); - } - - populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise { - return this.impl.populatePersistentChecksumCache(options); - } - - watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable { - return this.impl.watchCheckpointChanges(options); - } - - getCheckpointChanges(options: GetCheckpointChangesOptions): Promise { - return this.impl.getCheckpointChanges(options); - } + return new MongoSyncBucketStorageV1(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts index 284872247..1efb42a4a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts @@ -26,7 +26,7 @@ import { } from '@powersync/service-core'; import * as timers from 'node:timers/promises'; import { mongoTableId } from '../../../utils/util.js'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import { SyncRuleDocument } from '../models.js'; import { LoadedSourceRecord, SourceRecordStore } from './SourceRecordStore.js'; import { MongoIdSequence } from '../MongoIdSequence.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts index 07ffc1e93..bcb40f44b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts @@ -6,7 +6,7 @@ import { PartialChecksumMap } from '@powersync/service-core'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import { MongoChecksumsV1Impl } from '../v1/MongoChecksumsV1.js'; import { MongoChecksumsV3Impl } from '../v3/MongoChecksumsV3.js'; import { diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts index 2a5a5bf79..e147dcdd6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts @@ -15,7 +15,7 @@ import { PartialChecksumMap, PartialOrFullChecksum } from '@powersync/service-core'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { StorageConfig } from '../models.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts index 95a7daa82..3716ce9ac 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts @@ -1,6 +1,6 @@ import { PopulateChecksumCacheResults } from '@powersync/service-core'; -import { VersionedPowerSyncMongo } from '../db.js'; -import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; +import type { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; import { MongoCompactorV1 } from '../v1/MongoCompactorV1.js'; import { MongoCompactorV3 } from '../v3/MongoCompactorV3.js'; import { BaseMongoCompactor, DirtyBucket, MongoCompactOptions } from './MongoCompactorBase.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts index 473ad9666..0ecd9b157 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts @@ -9,7 +9,7 @@ import { utils } from '@powersync/service-core'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { BucketDataDocumentV1, @@ -18,7 +18,7 @@ import { TaggedBucketDataDocument, bucketDataDocumentToTagged } from '../models.js'; -import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; +import type { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; import { cacheKey } from '../OperationBatch.js'; interface CurrentBucketState { diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts index f2e2a1f17..06d17ecd6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts @@ -1,5 +1,5 @@ import { CompactOptions, InternalOpId } from '@powersync/service-core'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import { MongoParameterCompactorV1 } from '../v1/MongoParameterCompactorV1.js'; import { MongoParameterCompactorV3 } from '../v3/MongoParameterCompactorV3.js'; import { BaseMongoParameterCompactor } from './MongoParameterCompactorBase.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts index eedd0cc6b..b1c3f29aa 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts @@ -2,7 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { logger } from '@powersync/lib-services-framework'; import { bson, CompactOptions, InternalOpId } from '@powersync/service-core'; import { LRUCache } from 'lru-cache'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; type ParameterCompactionReadDocument = { _id: InternalOpId; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts index e7fbd8865..cf5a87f96 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts @@ -27,7 +27,7 @@ import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; import { idPrefixFilter, retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import { BucketDataKeyV1, BucketStateDocument, diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts index a6485b284..9e39b6bd0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts @@ -1,6 +1,6 @@ import { InternalOpId } from '@powersync/service-core'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import * as bson from 'bson'; export interface MongoSyncBucketStorageContext { diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index cb6003dfb..46bfbbae6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -5,7 +5,7 @@ import * as bson from 'bson'; import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { MongoIdSequence } from '../MongoIdSequence.js'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { TaggedBucketParameterDocument, TaggedBucketDataDocument } from '../models.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts new file mode 100644 index 000000000..c4847ba0d --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts @@ -0,0 +1,136 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; +import { PowerSyncMongo } from '../db.js'; +import { + BucketParameterDocumentV3, + BucketStateDocumentV3, + CommonSourceTableDocument, + CurrentDataDocument, + CurrentDataDocumentV3, + SourceTableDocumentV3, + StorageConfig +} from '../models.js'; + +export abstract class BaseVersionedPowerSyncMongo { + readonly client: mongo.MongoClient; + readonly db: mongo.Db; + readonly storageConfig: StorageConfig; + + constructor( + protected readonly upstream: PowerSyncMongo, + storageConfig: StorageConfig + ) { + this.client = upstream.client; + this.db = upstream.db; + this.storageConfig = storageConfig; + } + + get bucket_data() { + return this.upstream.bucket_data; + } + + get op_id_sequence() { + return this.upstream.op_id_sequence; + } + + get sync_rules() { + return this.upstream.sync_rules; + } + + get custom_write_checkpoints() { + return this.upstream.custom_write_checkpoints; + } + + get write_checkpoints() { + return this.upstream.write_checkpoints; + } + + get instance() { + return this.upstream.instance; + } + + get locks() { + return this.upstream.locks; + } + + get checkpoint_events() { + return this.upstream.checkpoint_events; + } + + get connection_report_events() { + return this.upstream.connection_report_events; + } + + notifyCheckpoint() { + return this.upstream.notifyCheckpoint(); + } + + protected assertV1Enabled(message: string) { + if (this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError(message); + } + } + + protected assertV3Enabled(message: string) { + if (!this.storageConfig.incrementalReprocessing) { + throw new ServiceAssertionError(message); + } + } + + protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId); + } + + protected sourceTableCollectionName(replicationStreamId: number) { + return this.upstream.sourceTableCollectionName(replicationStreamId); + } + + protected async listCollectionsByPrefix(prefix: string): Promise[]> { + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => this.db.collection(collection.name)); + } + + abstract get sourceRecordsV1(): mongo.Collection; + + abstract get bucketStateV1(): mongo.Collection; + + abstract sourceRecordsV3( + replicationStreamId: number, + sourceTableId: mongo.ObjectId + ): mongo.Collection; + + abstract listSourceRecordCollectionsV3( + replicationStreamId: number + ): Promise[]>; + + abstract initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId): Promise; + + abstract commonSourceTables(replicationStreamId: number): mongo.Collection; + + abstract bucketStateV3(replicationStreamId: number): mongo.Collection; + + abstract sourceTablesV3(replicationStreamId: number): mongo.Collection; + + abstract initializeStreamStorage(replicationStreamId: number): Promise; + + abstract get v1_bucket_data(): mongo.Collection; + + abstract bucket_data_v3(groupId: number, definitionId: BucketDefinitionId): mongo.Collection; + + abstract listBucketDataCollectionsV3(groupId: number): Promise[]>; + + abstract get parameterIndexV1(): mongo.Collection; + + abstract parameterIndexV3( + replicationStreamId: number, + indexId: ParameterIndexId + ): mongo.Collection; + + abstract listParameterIndexCollectionsV3( + replicationStreamId: number + ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]>; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index ee62b0e9f..b4b0c2b56 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -2,9 +2,9 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { POWERSYNC_VERSION, storage } from '@powersync/service-core'; -import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { MongoStorageConfig } from '../../types/types.js'; import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; +import { BaseVersionedPowerSyncMongo } from './common/VersionedPowerSyncMongoBase.js'; import { BucketDataDocumentV1, BucketDataDocumentV3, @@ -27,6 +27,8 @@ import { SyncRuleDocument, WriteCheckpointDocument } from './models.js'; +import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js'; +import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js'; export interface PowerSyncMongoOptions { /** @@ -76,8 +78,12 @@ export class PowerSyncMongo { this.connection_report_events = this.db.collection('connection_report_events'); } - versioned(storageConfig: StorageConfig) { - return new VersionedPowerSyncMongo(this, storageConfig); + versioned(storageConfig: StorageConfig): VersionedPowerSyncMongo { + if (storageConfig.incrementalReprocessing) { + return new VersionedPowerSyncMongoV3(this, storageConfig); + } + + return new VersionedPowerSyncMongoV1(this, storageConfig); } bucketDataCollectionNameV3(groupId: number, definitionId: BucketDefinitionId) { @@ -296,248 +302,7 @@ export class PowerSyncMongo { /** * This is similar to PowerSyncMongo, but blocks access to certain collections based on the storage version. */ -export class VersionedPowerSyncMongo { - readonly client: mongo.MongoClient; - readonly db: mongo.Db; - - readonly storageConfig: StorageConfig; - #upstream: PowerSyncMongo; - - constructor(upstream: PowerSyncMongo, storageConfig: StorageConfig) { - this.#upstream = upstream; - this.client = upstream.client; - this.db = upstream.db; - this.storageConfig = storageConfig; - } - - get sourceRecordsV1() { - if (this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'current_data collection should not be used when incrementalReprocessing is enabled' - ); - } - return this.#upstream.current_data; - } - - get bucketStateV1() { - if (this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'bucket_state collection should not be used when incrementalReprocessing is enabled' - ); - } - return this.#upstream.bucket_state; - } - - sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId) { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'v3_current_data collection should not be used when incrementalReprocessing is disabled' - ); - } - - const collectionName = `source_records_${replicationStreamId}_${sourceTableId.toHexString()}`; - return this.db.collection(collectionName); - } - - async listSourceRecordCollectionsV3(replicationStreamId: number): Promise[]> { - const prefix = `source_records_${replicationStreamId}_`; - const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); - - return collections - .filter((collection) => collection.name.startsWith(prefix)) - .map((collection) => this.db.collection(collection.name)); - } - - async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'source_records collection initialization should not be used when incrementalReprocessing is disabled' - ); - } - await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex( - { - pending_delete: 1 - }, - { - partialFilterExpression: { pending_delete: { $exists: true } }, - name: 'pending_delete' - } - ); - } - - commonSourceTables(replicationStreamId: number): mongo.Collection { - if (this.storageConfig.incrementalReprocessing) { - return this.sourceTablesV3(replicationStreamId) as mongo.Collection; - } else { - return this.#upstream.source_tables as any as mongo.Collection; - } - } - - bucketStateV3(replicationStreamId: number) { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'v3 bucket_state collection should not be used when incrementalReprocessing is disabled' - ); - } - return this.#upstream.bucketStateV3(replicationStreamId); - } - - sourceTablesV3(replicationStreamId: number) { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'source_tables v3 collection should not be used when incrementalReprocessing is disabled' - ); - } - return this.db.collection(this.#upstream.sourceTableCollectionName(replicationStreamId)); - } - - async initializeStreamStorage(replicationStreamId: number) { - if (this.storageConfig.incrementalReprocessing) { - const sourceTables = this.sourceTablesV3(replicationStreamId); - const bucketState = this.bucketStateV3(replicationStreamId); - await sourceTables.createIndex( - { - connection_id: 1, - schema_name: 1, - table_name: 1, - relation_id: 1 - }, - { - name: 'source_lookup' - } - ); - await sourceTables.createIndex( - { - latest_pending_delete: 1 - }, - { - partialFilterExpression: { latest_pending_delete: { $exists: true } }, - name: 'latest_pending_delete' - } - ); - await bucketState.createIndex( - { - last_op: 1 - }, - { name: 'bucket_updates', unique: true } - ); - await bucketState.createIndex( - { - 'estimate_since_compact.count': -1 - }, - { name: 'dirty_count' } - ); - } - } - - get bucket_data() { - return this.#upstream.bucket_data; - } - - get v1_bucket_data() { - if (this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'bucket_data collection should not be used when incrementalReprocessing is enabled' - ); - } - return this.#upstream.bucket_data; - } - - bucket_data_v3(groupId: number, definitionId: BucketDefinitionId) { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'v3 bucket_data collections should not be used when incrementalReprocessing is disabled' - ); - } - return this.#upstream.bucketDataV3(groupId, definitionId); - } - - listBucketDataCollectionsV3(groupId: number) { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'v3 bucket_data collections should not be used when incrementalReprocessing is disabled' - ); - } - return this.#upstream.listBucketDataCollectionsV3(groupId); - } - - get parameterIndexV1() { - if (this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'bucket_parameters collection should not be used when incrementalReprocessing is enabled' - ); - } - return this.#upstream.bucket_parameters; - } - - parameterIndexV3(replicationStreamId: number, indexId: ParameterIndexId) { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' - ); - } - return this.#upstream.parameterIndexV3(replicationStreamId, indexId); - } - - /** - * List parameter index collections for a specific replication stream. - */ - async listParameterIndexCollectionsV3( - replicationStreamId: number - ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]> { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError( - 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' - ); - } - - const prefix = `parameter_index_${replicationStreamId}_`; - const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); - - return collections - .filter((collection) => collection.name.startsWith(prefix)) - .map((collection) => ({ - collection: this.db.collection(collection.name), - indexId: collection.name.slice(prefix.length) - })); - } - - get op_id_sequence() { - return this.#upstream.op_id_sequence; - } - - get sync_rules() { - return this.#upstream.sync_rules; - } - - get custom_write_checkpoints() { - return this.#upstream.custom_write_checkpoints; - } - - get write_checkpoints() { - return this.#upstream.write_checkpoints; - } - - get instance() { - return this.#upstream.instance; - } - - get locks() { - return this.#upstream.locks; - } - - get checkpoint_events() { - return this.#upstream.checkpoint_events; - } - - get connection_report_events() { - return this.#upstream.connection_report_events; - } - - notifyCheckpoint() { - return this.#upstream.notifyCheckpoint(); - } -} +export type VersionedPowerSyncMongo = BaseVersionedPowerSyncMongo; export function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions) { return new PowerSyncMongo( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts index bca3188b8..a4b141c9c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts @@ -3,7 +3,7 @@ import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import * as bson from 'bson'; import { idPrefixFilter } from '../../../utils/util.js'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import { cacheKey } from '../OperationBatch.js'; import { SourceRecordLookupEntry, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts new file mode 100644 index 000000000..4f4fe3469 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts @@ -0,0 +1,101 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; +import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; +import { + BucketDataDocumentV1, + BucketParameterDocument, + BucketParameterDocumentV3, + BucketStateDocumentV1, + BucketStateDocumentV3, + CommonSourceTableDocument, + CurrentDataDocument, + CurrentDataDocumentV3, + SourceTableDocumentV3 +} from '../models.js'; + +export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo { + get sourceRecordsV1(): mongo.Collection { + this.assertV1Enabled('current_data collection should not be used when incrementalReprocessing is enabled'); + return this.upstream.current_data; + } + + get bucketStateV1(): mongo.Collection { + this.assertV1Enabled('bucket_state collection should not be used when incrementalReprocessing is enabled'); + return this.upstream.bucket_state; + } + + sourceRecordsV3( + _replicationStreamId: number, + _sourceTableId: mongo.ObjectId + ): mongo.Collection { + this.assertV3Enabled('v3_current_data collection should not be used when incrementalReprocessing is disabled'); + return this.db.collection('__invalid__'); + } + + listSourceRecordCollectionsV3(_replicationStreamId: number): Promise[]> { + this.assertV3Enabled('v3_current_data collection should not be used when incrementalReprocessing is disabled'); + return Promise.resolve([]); + } + + initializeSourceRecordsCollection(_replicationStreamId: number, _sourceTableId: mongo.ObjectId): Promise { + this.assertV3Enabled( + 'source_records collection initialization should not be used when incrementalReprocessing is disabled' + ); + return Promise.resolve(); + } + + commonSourceTables(_replicationStreamId: number): mongo.Collection { + return this.upstream.source_tables as any as mongo.Collection; + } + + bucketStateV3(_replicationStreamId: number): mongo.Collection { + this.assertV3Enabled('v3 bucket_state collection should not be used when incrementalReprocessing is disabled'); + return this.db.collection('__invalid__'); + } + + sourceTablesV3(_replicationStreamId: number): mongo.Collection { + this.assertV3Enabled('source_tables v3 collection should not be used when incrementalReprocessing is disabled'); + return this.db.collection('__invalid__'); + } + + async initializeStreamStorage(_replicationStreamId: number): Promise {} + + get v1_bucket_data(): mongo.Collection { + this.assertV1Enabled('bucket_data collection should not be used when incrementalReprocessing is enabled'); + return this.upstream.bucket_data; + } + + bucket_data_v3(_groupId: number, _definitionId: BucketDefinitionId): mongo.Collection { + this.assertV3Enabled('v3 bucket_data collections should not be used when incrementalReprocessing is disabled'); + return this.db.collection('__invalid__'); + } + + listBucketDataCollectionsV3(_groupId: number): Promise[]> { + this.assertV3Enabled('v3 bucket_data collections should not be used when incrementalReprocessing is disabled'); + return Promise.resolve([]); + } + + get parameterIndexV1(): mongo.Collection { + this.assertV1Enabled('bucket_parameters collection should not be used when incrementalReprocessing is enabled'); + return this.upstream.bucket_parameters; + } + + parameterIndexV3( + _replicationStreamId: number, + _indexId: ParameterIndexId + ): mongo.Collection { + this.assertV3Enabled( + 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' + ); + return this.db.collection('__invalid__'); + } + + listParameterIndexCollectionsV3( + _replicationStreamId: number + ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]> { + this.assertV3Enabled( + 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' + ); + return Promise.resolve([]); + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index 77bd61781..df8821ae5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -14,7 +14,7 @@ import { FetchPartialBucketChecksumV3, MongoChecksumOptions } from '../common/MongoChecksumsBase.js'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; export class MongoChecksumsV3Impl extends AbstractMongoChecksums { constructor( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts index 9f5a5a03a..7b0572828 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts @@ -5,7 +5,7 @@ import { storage } from '@powersync/service-core'; import * as bson from 'bson'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; -import { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from '../db.js'; import { cacheKey } from '../OperationBatch.js'; import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js'; import { CurrentDataDocumentV3, SourceTableDocumentV3 } from '../models.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts new file mode 100644 index 000000000..4ae9969c9 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -0,0 +1,146 @@ +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; +import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; +import { + BucketParameterDocumentV3, + BucketStateDocumentV3, + CommonSourceTableDocument, + CurrentDataDocument, + CurrentDataDocumentV3, + SourceTableDocumentV3 +} from '../models.js'; + +export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { + get sourceRecordsV1(): mongo.Collection { + this.assertV1Enabled('current_data collection should not be used when incrementalReprocessing is enabled'); + return this.db.collection('__invalid__'); + } + + get bucketStateV1(): mongo.Collection { + this.assertV1Enabled('bucket_state collection should not be used when incrementalReprocessing is enabled'); + return this.db.collection('__invalid__'); + } + + sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection { + this.assertV3Enabled('v3_current_data collection should not be used when incrementalReprocessing is disabled'); + const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId); + return this.db.collection(collectionName); + } + + async listSourceRecordCollectionsV3(replicationStreamId: number): Promise[]> { + this.assertV3Enabled('v3_current_data collection should not be used when incrementalReprocessing is disabled'); + return this.listCollectionsByPrefix(`source_records_${replicationStreamId}_`); + } + + async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { + this.assertV3Enabled( + 'source_records collection initialization should not be used when incrementalReprocessing is disabled' + ); + await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex( + { + pending_delete: 1 + }, + { + partialFilterExpression: { pending_delete: { $exists: true } }, + name: 'pending_delete' + } + ); + } + + commonSourceTables(replicationStreamId: number): mongo.Collection { + return this.sourceTablesV3(replicationStreamId) as mongo.Collection; + } + + bucketStateV3(replicationStreamId: number): mongo.Collection { + this.assertV3Enabled('v3 bucket_state collection should not be used when incrementalReprocessing is disabled'); + return this.upstream.bucketStateV3(replicationStreamId); + } + + sourceTablesV3(replicationStreamId: number): mongo.Collection { + this.assertV3Enabled('source_tables v3 collection should not be used when incrementalReprocessing is disabled'); + return this.db.collection(this.sourceTableCollectionName(replicationStreamId)); + } + + async initializeStreamStorage(replicationStreamId: number) { + const sourceTables = this.sourceTablesV3(replicationStreamId); + const bucketState = this.bucketStateV3(replicationStreamId); + await sourceTables.createIndex( + { + connection_id: 1, + schema_name: 1, + table_name: 1, + relation_id: 1 + }, + { + name: 'source_lookup' + } + ); + await sourceTables.createIndex( + { + latest_pending_delete: 1 + }, + { + partialFilterExpression: { latest_pending_delete: { $exists: true } }, + name: 'latest_pending_delete' + } + ); + await bucketState.createIndex( + { + last_op: 1 + }, + { name: 'bucket_updates', unique: true } + ); + await bucketState.createIndex( + { + 'estimate_since_compact.count': -1 + }, + { name: 'dirty_count' } + ); + } + + get v1_bucket_data(): mongo.Collection { + this.assertV1Enabled('bucket_data collection should not be used when incrementalReprocessing is enabled'); + return this.db.collection('__invalid__'); + } + + bucket_data_v3(groupId: number, definitionId: BucketDefinitionId) { + this.assertV3Enabled('v3 bucket_data collections should not be used when incrementalReprocessing is disabled'); + return this.upstream.bucketDataV3(groupId, definitionId); + } + + listBucketDataCollectionsV3(groupId: number) { + this.assertV3Enabled('v3 bucket_data collections should not be used when incrementalReprocessing is disabled'); + return this.upstream.listBucketDataCollectionsV3(groupId); + } + + get parameterIndexV1(): mongo.Collection { + this.assertV1Enabled('bucket_parameters collection should not be used when incrementalReprocessing is enabled'); + return this.db.collection('__invalid__'); + } + + parameterIndexV3(replicationStreamId: number, indexId: ParameterIndexId) { + this.assertV3Enabled( + 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' + ); + return this.upstream.parameterIndexV3(replicationStreamId, indexId); + } + + async listParameterIndexCollectionsV3( + replicationStreamId: number + ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]> { + this.assertV3Enabled( + 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' + ); + + const prefix = `parameter_index_${replicationStreamId}_`; + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); + + return collections + .filter((collection) => collection.name.startsWith(prefix)) + .map((collection) => ({ + collection: this.db.collection(collection.name), + indexId: collection.name.slice(prefix.length) + })); + } +} From 37f7e2cc7c3bd0ab5ecc86895260f6ea31048f70 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 17:34:46 +0200 Subject: [PATCH 61/93] Cleanup. --- .../implementation/common/MongoChecksums.ts | 3 +- .../common/MongoSyncBucketStorageBase.ts | 8 +-- .../common/MongoSyncBucketStorageContext.ts | 4 +- .../common/VersionedPowerSyncMongoBase.ts | 60 +--------------- .../implementation/v1/MongoBucketBatchV1.ts | 3 + .../implementation/v1/MongoChecksumsV1.ts | 5 +- .../implementation/v1/MongoCompactorV1.ts | 3 + .../v1/MongoParameterCompactorV1.ts | 3 + .../v1/MongoSyncBucketStorageV1.ts | 21 ++++-- .../implementation/v1/PersistedBatchV1.ts | 3 + .../implementation/v1/SourceRecordStoreV1.ts | 4 +- .../v1/VersionedPowerSyncMongoV1.ts | 70 +------------------ .../implementation/v3/MongoBucketBatchV3.ts | 3 + .../implementation/v3/MongoChecksumsV3.ts | 6 +- .../implementation/v3/MongoCompactorV3.ts | 3 + .../v3/MongoParameterCompactorV3.ts | 3 + .../v3/MongoSyncBucketStorageV3.ts | 25 +++++-- .../implementation/v3/PersistedBatchV3.ts | 3 + .../implementation/v3/SourceRecordStoreV3.ts | 4 +- .../v3/VersionedPowerSyncMongoV3.ts | 37 ---------- .../test/src/storage_sync.test.ts | 6 +- 21 files changed, 86 insertions(+), 191 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts index bcb40f44b..2bd48d45e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts @@ -9,6 +9,7 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework'; import type { VersionedPowerSyncMongo } from '../db.js'; import { MongoChecksumsV1Impl } from '../v1/MongoChecksumsV1.js'; import { MongoChecksumsV3Impl } from '../v3/MongoChecksumsV3.js'; +import type { VersionedPowerSyncMongoV3 } from '../v3/VersionedPowerSyncMongoV3.js'; import { AbstractMongoChecksums, FetchPartialBucketChecksumByBucket, @@ -33,7 +34,7 @@ export class MongoChecksums { constructor(db: VersionedPowerSyncMongo, group_id: number, options: MongoChecksumOptions) { if (options.storageConfig.incrementalReprocessing) { this.v3Impl = new MongoChecksumsV3Impl( - db, + db as VersionedPowerSyncMongoV3, group_id, options, options.mapping ?? diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts index cf5a87f96..3c9c29380 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts @@ -435,6 +435,8 @@ export abstract class BaseMongoSyncBucketStorage protected abstract clearParameterIndexes(signal?: AbortSignal): Promise; + protected abstract clearSourceRecords(signal?: AbortSignal): Promise; + protected abstract clearBucketState(signal?: AbortSignal): Promise; protected abstract clearSourceTables(signal?: AbortSignal): Promise; @@ -467,11 +469,7 @@ export abstract class BaseMongoSyncBucketStorage await this.clearBucketData(signal); await this.clearParameterIndexes(signal); - - for (const collection of await this.db.listSourceRecordCollectionsV3(this.group_id)) { - await collection.drop(); - } - + await this.clearSourceRecords(signal); await this.clearBucketState(signal); await this.clearSourceTables(signal); diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts index 9e39b6bd0..a5965069a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts @@ -3,8 +3,8 @@ import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import type { VersionedPowerSyncMongo } from '../db.js'; import * as bson from 'bson'; -export interface MongoSyncBucketStorageContext { - db: VersionedPowerSyncMongo; +export interface MongoSyncBucketStorageContext { + db: TDb; group_id: number; mapping: BucketDefinitionMapping; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts index c4847ba0d..f5c911d65 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts @@ -1,16 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; import { PowerSyncMongo } from '../db.js'; -import { - BucketParameterDocumentV3, - BucketStateDocumentV3, - CommonSourceTableDocument, - CurrentDataDocument, - CurrentDataDocumentV3, - SourceTableDocumentV3, - StorageConfig -} from '../models.js'; +import { CommonSourceTableDocument, StorageConfig } from '../models.js'; export abstract class BaseVersionedPowerSyncMongo { readonly client: mongo.MongoClient; @@ -66,18 +56,6 @@ export abstract class BaseVersionedPowerSyncMongo { return this.upstream.notifyCheckpoint(); } - protected assertV1Enabled(message: string) { - if (this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError(message); - } - } - - protected assertV3Enabled(message: string) { - if (!this.storageConfig.incrementalReprocessing) { - throw new ServiceAssertionError(message); - } - } - protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) { return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId); } @@ -94,43 +72,7 @@ export abstract class BaseVersionedPowerSyncMongo { .map((collection) => this.db.collection(collection.name)); } - abstract get sourceRecordsV1(): mongo.Collection; - - abstract get bucketStateV1(): mongo.Collection; - - abstract sourceRecordsV3( - replicationStreamId: number, - sourceTableId: mongo.ObjectId - ): mongo.Collection; - - abstract listSourceRecordCollectionsV3( - replicationStreamId: number - ): Promise[]>; - - abstract initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId): Promise; - abstract commonSourceTables(replicationStreamId: number): mongo.Collection; - abstract bucketStateV3(replicationStreamId: number): mongo.Collection; - - abstract sourceTablesV3(replicationStreamId: number): mongo.Collection; - abstract initializeStreamStorage(replicationStreamId: number): Promise; - - abstract get v1_bucket_data(): mongo.Collection; - - abstract bucket_data_v3(groupId: number, definitionId: BucketDefinitionId): mongo.Collection; - - abstract listBucketDataCollectionsV3(groupId: number): Promise[]>; - - abstract get parameterIndexV1(): mongo.Collection; - - abstract parameterIndexV3( - replicationStreamId: number, - indexId: ParameterIndexId - ): mongo.Collection; - - abstract listParameterIndexCollectionsV3( - replicationStreamId: number - ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]>; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts index 8e10dce27..277a2cea8 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts @@ -3,8 +3,11 @@ import { SourceRecordStore } from '../common/SourceRecordStore.js'; import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; import { PersistedBatchV1 } from './PersistedBatchV1.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoBucketBatchV1 extends MongoBucketBatch { + declare public readonly db: VersionedPowerSyncMongoV1; + private readonly store: SourceRecordStore; constructor(options: MongoBucketBatchOptions) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts index 5bb2a6c03..64e1ae342 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts @@ -7,14 +7,17 @@ import { PartialChecksumMap } from '@powersync/service-core'; import { AbstractMongoChecksums, FetchPartialBucketChecksumByBucket } from '../common/MongoChecksumsBase.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoChecksumsV1Impl extends AbstractMongoChecksums { + declare protected readonly db: VersionedPowerSyncMongoV1; + async computePartialChecksumsDirectByBucket( batch: FetchPartialBucketChecksumByBucket[] ): Promise { return this.computePartialChecksumsForCollection( batch, - this.db.bucket_data as unknown as mongo.Collection, + this.db.v1_bucket_data as unknown as mongo.Collection, (request) => ({ _id: { $gt: { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 791f3514b..9c4a2409f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -10,8 +10,11 @@ import { } from '../models.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { BaseMongoCompactor, DirtyBucket } from '../common/MongoCompactorBase.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoCompactorV1 extends BaseMongoCompactor { + declare protected readonly db: VersionedPowerSyncMongoV1; + public async *dirtyBucketBatches(options: { minBucketChanges: number; minChangeRatio: number; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts index 224c9dd2c..be6cb90e9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts @@ -1,7 +1,10 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { BaseMongoParameterCompactor } from '../common/MongoParameterCompactorBase.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoParameterCompactorV1 extends BaseMongoParameterCompactor { + declare protected readonly db: VersionedPowerSyncMongoV1; + protected async getCollections(): Promise[]> { return [this.db.parameterIndexV1 as unknown as mongo.Collection]; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 66703552a..35a134762 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -31,8 +31,11 @@ import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { + declare readonly db: VersionedPowerSyncMongoV1; + constructor( factory: MongoBucketStorage, group_id: number, @@ -62,6 +65,14 @@ export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { protected async initializeResolvedSourceRecords(_sourceTableId: bson.ObjectId): Promise {} + protected override get versionContext(): MongoSyncBucketStorageContext { + return { + db: this.db, + group_id: this.group_id, + mapping: this.mapping + }; + } + protected getParameterSetsImpl( checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[] @@ -105,6 +116,8 @@ export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { ); } + protected async clearSourceRecords(_signal?: AbortSignal): Promise {} + protected async clearBucketState(signal?: AbortSignal): Promise { await this.clearDeleteMany( 'bucket state', @@ -147,7 +160,7 @@ export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { } export async function getParameterSetsV1( - ctx: MongoSyncBucketStorageContext, + ctx: MongoSyncBucketStorageContext, checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[] ): Promise { @@ -198,7 +211,7 @@ export async function getParameterSetsV1( } export async function* getBucketDataBatchV1( - ctx: MongoSyncBucketStorageContext, + ctx: MongoSyncBucketStorageContext, checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions @@ -316,7 +329,7 @@ export async function* getBucketDataBatchV1( } export async function getDataBucketChangesV1( - ctx: MongoSyncBucketStorageContext, + ctx: MongoSyncBucketStorageContext, options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; @@ -347,7 +360,7 @@ export async function getDataBucketChangesV1( } export async function getParameterBucketChangesV1( - ctx: MongoSyncBucketStorageContext, + ctx: MongoSyncBucketStorageContext, options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index 9420e1b82..33f19bfc6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -23,8 +23,11 @@ import { } from '../models.js'; import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; import { BucketStateUpdate } from '../common/PersistedBatch.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class PersistedBatchV1 extends PersistedBatch { + declare protected readonly db: VersionedPowerSyncMongoV1; + currentData: mongo.AnyBulkWriteOperation[] = []; saveBucketData(options: SaveBucketDataOptions) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts index a4b141c9c..42d6eafcd 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts @@ -3,7 +3,6 @@ import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import * as bson from 'bson'; import { idPrefixFilter } from '../../../utils/util.js'; -import type { VersionedPowerSyncMongo } from '../db.js'; import { cacheKey } from '../OperationBatch.js'; import { SourceRecordLookupEntry, @@ -13,10 +12,11 @@ import { } from '../common/SourceRecordStore.js'; import { CurrentDataDocument, SourceKey } from '../models.js'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class SourceRecordStoreV1 implements SourceRecordStore { constructor( - private readonly db: VersionedPowerSyncMongo, + private readonly db: VersionedPowerSyncMongoV1, private readonly groupId: number ) {} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts index 4f4fe3469..6d4ebc9e9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts @@ -1,101 +1,33 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; import { BucketDataDocumentV1, BucketParameterDocument, - BucketParameterDocumentV3, BucketStateDocumentV1, - BucketStateDocumentV3, CommonSourceTableDocument, - CurrentDataDocument, - CurrentDataDocumentV3, - SourceTableDocumentV3 + CurrentDataDocument } from '../models.js'; export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo { get sourceRecordsV1(): mongo.Collection { - this.assertV1Enabled('current_data collection should not be used when incrementalReprocessing is enabled'); return this.upstream.current_data; } get bucketStateV1(): mongo.Collection { - this.assertV1Enabled('bucket_state collection should not be used when incrementalReprocessing is enabled'); return this.upstream.bucket_state; } - sourceRecordsV3( - _replicationStreamId: number, - _sourceTableId: mongo.ObjectId - ): mongo.Collection { - this.assertV3Enabled('v3_current_data collection should not be used when incrementalReprocessing is disabled'); - return this.db.collection('__invalid__'); - } - - listSourceRecordCollectionsV3(_replicationStreamId: number): Promise[]> { - this.assertV3Enabled('v3_current_data collection should not be used when incrementalReprocessing is disabled'); - return Promise.resolve([]); - } - - initializeSourceRecordsCollection(_replicationStreamId: number, _sourceTableId: mongo.ObjectId): Promise { - this.assertV3Enabled( - 'source_records collection initialization should not be used when incrementalReprocessing is disabled' - ); - return Promise.resolve(); - } - commonSourceTables(_replicationStreamId: number): mongo.Collection { return this.upstream.source_tables as any as mongo.Collection; } - bucketStateV3(_replicationStreamId: number): mongo.Collection { - this.assertV3Enabled('v3 bucket_state collection should not be used when incrementalReprocessing is disabled'); - return this.db.collection('__invalid__'); - } - - sourceTablesV3(_replicationStreamId: number): mongo.Collection { - this.assertV3Enabled('source_tables v3 collection should not be used when incrementalReprocessing is disabled'); - return this.db.collection('__invalid__'); - } - async initializeStreamStorage(_replicationStreamId: number): Promise {} get v1_bucket_data(): mongo.Collection { - this.assertV1Enabled('bucket_data collection should not be used when incrementalReprocessing is enabled'); return this.upstream.bucket_data; } - bucket_data_v3(_groupId: number, _definitionId: BucketDefinitionId): mongo.Collection { - this.assertV3Enabled('v3 bucket_data collections should not be used when incrementalReprocessing is disabled'); - return this.db.collection('__invalid__'); - } - - listBucketDataCollectionsV3(_groupId: number): Promise[]> { - this.assertV3Enabled('v3 bucket_data collections should not be used when incrementalReprocessing is disabled'); - return Promise.resolve([]); - } - get parameterIndexV1(): mongo.Collection { - this.assertV1Enabled('bucket_parameters collection should not be used when incrementalReprocessing is enabled'); return this.upstream.bucket_parameters; } - - parameterIndexV3( - _replicationStreamId: number, - _indexId: ParameterIndexId - ): mongo.Collection { - this.assertV3Enabled( - 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' - ); - return this.db.collection('__invalid__'); - } - - listParameterIndexCollectionsV3( - _replicationStreamId: number - ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]> { - this.assertV3Enabled( - 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' - ); - return Promise.resolve([]); - } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts index 5d39bcc55..9d0af9e5e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -6,8 +6,11 @@ import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; import { PersistedBatchV3 } from './PersistedBatchV3.js'; import { mongoTableId } from '../../../utils/util.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoBucketBatchV3 extends MongoBucketBatch { + declare public readonly db: VersionedPowerSyncMongoV3; + private readonly store: SourceRecordStore; constructor(options: MongoBucketBatchOptions) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index df8821ae5..9dd0bb519 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -14,11 +14,13 @@ import { FetchPartialBucketChecksumV3, MongoChecksumOptions } from '../common/MongoChecksumsBase.js'; -import type { VersionedPowerSyncMongo } from '../db.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoChecksumsV3Impl extends AbstractMongoChecksums { + declare protected readonly db: VersionedPowerSyncMongoV3; + constructor( - db: VersionedPowerSyncMongo, + db: VersionedPowerSyncMongoV3, group_id: number, options: MongoChecksumOptions, private readonly mapping: BucketDefinitionMapping diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 7c4a0f85e..7a8b96673 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -9,8 +9,11 @@ import { } from '../models.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { BaseMongoCompactor, DirtyBucket } from '../common/MongoCompactorBase.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoCompactorV3 extends BaseMongoCompactor { + declare protected readonly db: VersionedPowerSyncMongoV3; + public async *dirtyBucketBatches(options: { minBucketChanges: number; minChangeRatio: number; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts index 65999d718..bce34750a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts @@ -1,7 +1,10 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { BaseMongoParameterCompactor } from '../common/MongoParameterCompactorBase.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoParameterCompactorV3 extends BaseMongoParameterCompactor { + declare protected readonly db: VersionedPowerSyncMongoV3; + protected async getCollections(): Promise[]> { const collections = await this.db.listParameterIndexCollectionsV3(this.group_id); return collections.map((collection) => collection.collection as unknown as mongo.Collection); diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 946cda41e..275999b52 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -25,8 +25,11 @@ import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesConten import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; import { CommonSourceTableDocument } from '../models.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { + declare readonly db: VersionedPowerSyncMongoV3; + constructor( factory: MongoBucketStorage, group_id: number, @@ -95,6 +98,14 @@ export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { await this.db.initializeSourceRecordsCollection(this.group_id, sourceTableId); } + protected override get versionContext(): MongoSyncBucketStorageContext { + return { + db: this.db, + group_id: this.group_id, + mapping: this.mapping + }; + } + protected getParameterSetsImpl( checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[] @@ -122,6 +133,12 @@ export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { } } + protected async clearSourceRecords(_signal?: AbortSignal): Promise { + for (const collection of await this.db.listSourceRecordCollectionsV3(this.group_id)) { + await collection.drop(); + } + } + protected async clearBucketState(_signal?: AbortSignal): Promise { await this.db .bucketStateV3(this.group_id) @@ -160,7 +177,7 @@ export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { } export async function getParameterSetsV3( - ctx: MongoSyncBucketStorageContext, + ctx: MongoSyncBucketStorageContext, checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[] ): Promise { @@ -246,7 +263,7 @@ export async function getParameterSetsV3( } export async function* getBucketDataBatchV3( - ctx: MongoSyncBucketStorageContext, + ctx: MongoSyncBucketStorageContext, checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions @@ -381,7 +398,7 @@ export async function* getBucketDataBatchV3( } export async function getDataBucketChangesV3( - ctx: MongoSyncBucketStorageContext, + ctx: MongoSyncBucketStorageContext, options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; @@ -423,7 +440,7 @@ export async function getDataBucketChangesV3( } export async function getParameterBucketChangesV3( - ctx: MongoSyncBucketStorageContext, + ctx: MongoSyncBucketStorageContext, options: GetCheckpointChangesOptions ): Promise> { const limit = 1000; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 6fb122ab7..804f54a52 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -23,8 +23,11 @@ import { } from '../models.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; import { BucketStateUpdate } from '../common/PersistedBatch.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class PersistedBatchV3 extends PersistedBatch { + declare protected readonly db: VersionedPowerSyncMongoV3; + currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation }[] = []; sourceTablePendingDeletes = new Map(); diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts index 7b0572828..b5a3fd2a5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts @@ -5,16 +5,16 @@ import { storage } from '@powersync/service-core'; import * as bson from 'bson'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; -import type { VersionedPowerSyncMongo } from '../db.js'; import { cacheKey } from '../OperationBatch.js'; import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js'; import { CurrentDataDocumentV3, SourceTableDocumentV3 } from '../models.js'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class SourceRecordStoreV3 implements SourceRecordStore { constructor( - private readonly db: VersionedPowerSyncMongo, + private readonly db: VersionedPowerSyncMongoV3, private readonly groupId: number, private readonly mapping: BucketDefinitionMapping ) {} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts index 4ae9969c9..3854bcff4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -6,37 +6,21 @@ import { BucketParameterDocumentV3, BucketStateDocumentV3, CommonSourceTableDocument, - CurrentDataDocument, CurrentDataDocumentV3, SourceTableDocumentV3 } from '../models.js'; export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { - get sourceRecordsV1(): mongo.Collection { - this.assertV1Enabled('current_data collection should not be used when incrementalReprocessing is enabled'); - return this.db.collection('__invalid__'); - } - - get bucketStateV1(): mongo.Collection { - this.assertV1Enabled('bucket_state collection should not be used when incrementalReprocessing is enabled'); - return this.db.collection('__invalid__'); - } - sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection { - this.assertV3Enabled('v3_current_data collection should not be used when incrementalReprocessing is disabled'); const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId); return this.db.collection(collectionName); } async listSourceRecordCollectionsV3(replicationStreamId: number): Promise[]> { - this.assertV3Enabled('v3_current_data collection should not be used when incrementalReprocessing is disabled'); return this.listCollectionsByPrefix(`source_records_${replicationStreamId}_`); } async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) { - this.assertV3Enabled( - 'source_records collection initialization should not be used when incrementalReprocessing is disabled' - ); await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex( { pending_delete: 1 @@ -53,12 +37,10 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { } bucketStateV3(replicationStreamId: number): mongo.Collection { - this.assertV3Enabled('v3 bucket_state collection should not be used when incrementalReprocessing is disabled'); return this.upstream.bucketStateV3(replicationStreamId); } sourceTablesV3(replicationStreamId: number): mongo.Collection { - this.assertV3Enabled('source_tables v3 collection should not be used when incrementalReprocessing is disabled'); return this.db.collection(this.sourceTableCollectionName(replicationStreamId)); } @@ -99,40 +81,21 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { ); } - get v1_bucket_data(): mongo.Collection { - this.assertV1Enabled('bucket_data collection should not be used when incrementalReprocessing is enabled'); - return this.db.collection('__invalid__'); - } - bucket_data_v3(groupId: number, definitionId: BucketDefinitionId) { - this.assertV3Enabled('v3 bucket_data collections should not be used when incrementalReprocessing is disabled'); return this.upstream.bucketDataV3(groupId, definitionId); } listBucketDataCollectionsV3(groupId: number) { - this.assertV3Enabled('v3 bucket_data collections should not be used when incrementalReprocessing is disabled'); return this.upstream.listBucketDataCollectionsV3(groupId); } - get parameterIndexV1(): mongo.Collection { - this.assertV1Enabled('bucket_parameters collection should not be used when incrementalReprocessing is enabled'); - return this.db.collection('__invalid__'); - } - parameterIndexV3(replicationStreamId: number, indexId: ParameterIndexId) { - this.assertV3Enabled( - 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' - ); return this.upstream.parameterIndexV3(replicationStreamId, indexId); } async listParameterIndexCollectionsV3( replicationStreamId: number ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]> { - this.assertV3Enabled( - 'v3 bucket_parameters collections should not be used when incrementalReprocessing is disabled' - ); - const prefix = `parameter_index_${replicationStreamId}_`; const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 8dc02e9ef..b8dbbc141 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -7,6 +7,7 @@ import { describe, expect, test } from 'vitest'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/MongoSyncBucketStorage.js'; import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js'; +import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; import { CurrentBucketV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -205,9 +206,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'global["user-1"]').bucket]); const mongoFactory = factory as MongoBucketStorage; - const currentDataCollections = await (bucketStorage as MongoSyncBucketStorage).db.listSourceRecordCollectionsV3( - syncRules.id - ); + const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3; + const currentDataCollections = await db.listSourceRecordCollectionsV3(syncRules.id); const currentData = await currentDataCollections[0]?.findOne({}); const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined; expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/); From 5d82cef340e5d309fce3ba965dc5dcea2d46d297 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 17:52:57 +0200 Subject: [PATCH 62/93] Tweak types. --- .../common/MongoCompactorBase.ts | 77 ++++++++----------- .../src/storage/implementation/models.ts | 10 ++- .../implementation/v1/MongoCompactorV1.ts | 20 +++-- .../implementation/v3/MongoCompactorV3.ts | 25 +++--- 4 files changed, 69 insertions(+), 63 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts index 0ecd9b157..70825d5d5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts @@ -12,10 +12,12 @@ import { import type { VersionedPowerSyncMongo } from '../db.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { + BucketDataDocumentBase, BucketDataDocumentV1, BucketDataDocumentV3, LEGACY_BUCKET_DATA_DEFINITION_ID, TaggedBucketDataDocument, + BucketStateDocumentBase, bucketDataDocumentToTagged } from '../models.js'; import type { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; @@ -65,24 +67,12 @@ type CompactBucketDataDocument = Pick< type CompactClearBucketDataDocument = Pick; type BucketDataCollectionDocument = BucketDataDocumentV1 | BucketDataDocumentV3; type BucketDataClearProjection = { - _id: BucketDataCollectionDocument['_id']; + _id: BucketDataDocumentBase['_id']; op: CompactClearBucketDataDocument['op']; checksum: bigint; target_op?: bigint | null; }; -type BucketStateProjection = { - _id: { b: string }; - estimate_since_compact?: { - count: number; - bytes: number | bigint; - }; - compacted_state?: { - count: number; - bytes: number | bigint | null; - }; -}; - export interface MongoCompactOptions extends storage.CompactOptions {} const DEFAULT_CLEAR_BATCH_LIMIT = 5000; @@ -102,9 +92,9 @@ export interface DirtyBucket { } export abstract class BaseMongoCompactor { - protected updates: mongo.AnyBulkWriteOperation[] = []; - protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; - protected activeBucketDataCollection: mongo.Collection | null = null; + protected updates: mongo.AnyBulkWriteOperation[] = []; + protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; + protected activeBucketDataCollection: mongo.Collection | null = null; protected activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; protected readonly idLimitBytes: number; @@ -186,23 +176,23 @@ export abstract class BaseMongoCompactor { return { buckets: count }; } - protected async *dirtyBucketBatchesForCollection( - collection: mongo.Collection, - lastId: mongo.Document, - maxId: mongo.Document, + protected async *dirtyBucketBatchesForCollection( + collection: mongo.Collection, + lastId: TCollectionBucketState['_id'], + maxId: TCollectionBucketState['_id'], options: { minBucketChanges: number; minChangeRatio: number; }, - getDefinitionId: (state: TBucketState) => BucketDefinitionId | null + getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null ): AsyncGenerator { while (true) { // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria. const [result] = await collection .aggregate<{ - buckets: TBucketState[]; - cursor: Pick[]; + buckets: TCollectionBucketState[]; + cursor: Pick[]; }>( [ { @@ -246,7 +236,7 @@ export abstract class BaseMongoCompactor { if (cursor == null) { break; } - lastId = cursor._id as mongo.Document; + lastId = cursor._id; const mapped = (result?.buckets ?? []).map((bucketState) => { // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios. @@ -271,7 +261,7 @@ export abstract class BaseMongoCompactor { } } - protected async dirtyBucketBatchForChecksumsForCollection( + protected async dirtyBucketBatchForChecksumsForCollection( collection: mongo.Collection, filter: mongo.Filter, getDefinitionId: (state: mongo.WithId) => BucketDefinitionId | null @@ -456,7 +446,7 @@ export abstract class BaseMongoCompactor { row_id: 1, data: 1 } - } + } satisfies mongo.UpdateFilter } }); @@ -536,7 +526,7 @@ export abstract class BaseMongoCompactor { bytes: 0 } } - }, + } satisfies mongo.UpdateFilter, // We generally expect this to have been created before. // We don't create new ones here, to avoid issues with the unique index on bucket_updates. upsert: false @@ -613,7 +603,7 @@ export abstract class BaseMongoCompactor { let numberOfOpsToClear = 0; for await (const rawOp of query.stream()) { const op = this.tagClearBucketDataDocument( - rawOp as unknown as BucketDataClearProjection, + rawOp as BucketDataClearProjection, this.activeBucketDefinitionId ); @@ -629,7 +619,7 @@ export abstract class BaseMongoCompactor { } } else { throw new ReplicationAssertionError( - `Unexpected ${op.op} operation at ${this.formatBucketDataKey(op._id as unknown as mongo.Document)}` + `Unexpected ${op.op} operation at ${this.formatBucketDataKey(op._id)}` ); } } @@ -642,11 +632,11 @@ export abstract class BaseMongoCompactor { await bucketCollection.deleteMany( { _id: { - $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), + $gte: this.bucketDataKey(bucket, new mongo.MinKey()), $lte: this.bucketDataKey(lastOp!._id.b, lastOp!._id.o) } - } as any, - { session } as any + }, + { session } ); await bucketCollection.insertOne( @@ -657,8 +647,8 @@ export abstract class BaseMongoCompactor { checksum: BigInt(checksum), data: null, target_op: targetOp - }) as unknown as mongo.OptionalId, - { session } as any + }), + { session } ); opCountDiff = -numberOfOpsToClear + 1; @@ -705,7 +695,7 @@ export abstract class BaseMongoCompactor { bytes: 0 } } - }, + } satisfies mongo.UpdateFilter, // We don't create new ones here - it gets tricky to get the last_op right with the unique index on // bucket_updates. upsert: false @@ -743,9 +733,9 @@ export abstract class BaseMongoCompactor { }; } - protected formatBucketDataKey(key: mongo.Document) { - const bucket = (key.b ?? key._id?.b) as string | undefined; - const op = (key.o ?? key._id?.o) as bigint | undefined; + protected formatBucketDataKey(key: BucketDataDocumentBase['_id'] | { _id: BucketDataDocumentBase['_id'] }) { + const bucket = 'b' in key ? key.b : key._id.b; + const op = 'o' in key ? key.o : key._id.o; return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; } @@ -754,12 +744,13 @@ export abstract class BaseMongoCompactor { buckets: Pick[] ): Promise; protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; - protected abstract bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document; + protected abstract bucketDataKey( + bucket: string, + opId: InternalOpId | mongo.MinKey | mongo.MaxKey + ): BucketDataDocumentBase['_id']; protected abstract getBucketDataCollection( bucket: string, definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null>; - protected abstract collectionBucketDataDocument( - document: TaggedBucketDataDocument - ): BucketDataDocumentV1 | BucketDataDocumentV3; + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null>; + protected abstract collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 07aa04e9e..9f64e8d97 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -111,11 +111,15 @@ export interface BucketDataProperties { target_op?: bigint | null; } -export interface BucketDataDocumentV1 extends BucketDataProperties { +export interface BucketDataDocumentBase extends BucketDataProperties { + _id: BucketDataKeyV3; +} + +export interface BucketDataDocumentV1 extends BucketDataDocumentBase { _id: BucketDataKeyV1; } -export interface BucketDataDocumentV3 extends BucketDataProperties { +export interface BucketDataDocumentV3 extends BucketDataDocumentBase { _id: BucketDataKeyV3; } @@ -231,7 +235,7 @@ export interface SourceTableDocumentSnapshotStatus { * Note: There is currently no migration to populate this collection from existing data - it is only * populated by new updates. */ -interface BucketStateDocumentBase { +export interface BucketStateDocumentBase { _id: { b: string; }; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 9c4a2409f..f933813fd 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -2,7 +2,10 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { + BucketDataDocumentBase, BucketDataDocumentV1, + BucketDataKeyV1, + BucketStateDocumentBase, BucketStateDocumentV1, LEGACY_BUCKET_DATA_DEFINITION_ID, TaggedBucketDataDocument, @@ -53,9 +56,7 @@ export class MongoCompactorV1 extends BaseMongoCompactor { protected async flushBucketStateUpdates(): Promise { await this.db.bucketStateV1.bulkWrite( this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], - { - ordered: false - } + { ordered: false } ); } @@ -70,7 +71,10 @@ export class MongoCompactorV1 extends BaseMongoCompactor { ); } - protected bucketStateFilter(bucket: string, _definitionId: BucketDefinitionId | null): mongo.Document { + protected bucketStateFilter( + bucket: string, + _definitionId: BucketDefinitionId | null + ): mongo.Filter { return { _id: { g: this.group_id, @@ -79,7 +83,7 @@ export class MongoCompactorV1 extends BaseMongoCompactor { }; } - protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document { + protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): BucketDataKeyV1 { return { g: this.group_id, b: bucket, @@ -90,14 +94,14 @@ export class MongoCompactorV1 extends BaseMongoCompactor { protected async getBucketDataCollection( _bucket: string, _definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { return { - collection: this.db.v1_bucket_data as unknown as mongo.Collection, + collection: this.db.v1_bucket_data as unknown as mongo.Collection, definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID }; } - protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentV1 { + protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase { return taggedBucketDataDocumentToV1(this.group_id, document); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 7a8b96673..ec9a5bd24 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -2,7 +2,10 @@ import { MONGO_OPERATION_TIMEOUT_MS, mongo } from '@powersync/lib-service-mongod import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { + BucketDataDocumentBase, BucketDataDocumentV3, + BucketDataKeyV3, + BucketStateDocumentBase, BucketStateDocumentV3, TaggedBucketDataDocument, taggedBucketDataDocumentToV3 @@ -47,9 +50,7 @@ export class MongoCompactorV3 extends BaseMongoCompactor { protected async flushBucketStateUpdates(): Promise { await this.db .bucketStateV3(this.group_id) - .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { - ordered: false - }); + .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { ordered: false }); } protected async computeChecksumsForBuckets( @@ -69,7 +70,10 @@ export class MongoCompactorV3 extends BaseMongoCompactor { ); } - protected bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document { + protected bucketStateFilter( + bucket: string, + definitionId: BucketDefinitionId | null + ): mongo.Filter { if (definitionId == null) { throw new ServiceAssertionError(`Missing definitionId for V3 bucket state filter on bucket ${bucket}`); } @@ -81,17 +85,20 @@ export class MongoCompactorV3 extends BaseMongoCompactor { }; } - protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): mongo.Document { + protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): BucketDataKeyV3 { return { b: bucket, o: opId as any }; } protected async getBucketDataCollection( bucket: string, definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { if (definitionId != null) { return { - collection: this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, + collection: this.db.bucket_data_v3( + this.group_id, + definitionId + ) as unknown as mongo.Collection, definitionId }; } @@ -105,7 +112,7 @@ export class MongoCompactorV3 extends BaseMongoCompactor { if (existing != null) { const resolvedDefinitionId = collection.collectionName.replace(`bucket_data_${this.group_id}_`, ''); return { - collection: collection as unknown as mongo.Collection, + collection: collection as unknown as mongo.Collection, definitionId: resolvedDefinitionId }; } @@ -114,7 +121,7 @@ export class MongoCompactorV3 extends BaseMongoCompactor { return null; } - protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentV3 { + protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase { return taggedBucketDataDocumentToV3(document); } } From 0d1420a818edcc1a54c33aad4caad53cf8d2249b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 31 Mar 2026 17:57:40 +0200 Subject: [PATCH 63/93] More type tweaks. --- .../common/MongoChecksumsBase.ts | 9 ++++-- .../implementation/v1/MongoChecksumsV1.ts | 31 +++++++++---------- .../implementation/v3/MongoChecksumsV3.ts | 3 +- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts index e147dcdd6..20d139468 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts @@ -17,7 +17,7 @@ import { } from '@powersync/service-core'; import type { VersionedPowerSyncMongo } from '../db.js'; import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; -import { StorageConfig } from '../models.js'; +import { BucketDataDocumentBase, StorageConfig } from '../models.js'; export interface FetchPartialBucketChecksumV3 { bucket: string; @@ -188,9 +188,12 @@ export abstract class AbstractMongoChecksums { batch: FetchPartialBucketChecksum[] ): Promise>; - protected async computePartialChecksumsForCollection( + protected async computePartialChecksumsForCollection< + TRequest extends FetchPartialBucketChecksumByBucket, + TBucketDataDocument extends BucketDataDocumentBase + >( batch: TRequest[], - collection: mongo.Collection, + collection: mongo.Collection, createFilter: (request: TRequest) => any ): Promise { const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts index 64e1ae342..cea25a5a1 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts @@ -8,6 +8,7 @@ import { } from '@powersync/service-core'; import { AbstractMongoChecksums, FetchPartialBucketChecksumByBucket } from '../common/MongoChecksumsBase.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; +import { BucketDataDocumentBase } from '../models.js'; export class MongoChecksumsV1Impl extends AbstractMongoChecksums { declare protected readonly db: VersionedPowerSyncMongoV1; @@ -15,24 +16,20 @@ export class MongoChecksumsV1Impl extends AbstractMongoChecksums { async computePartialChecksumsDirectByBucket( batch: FetchPartialBucketChecksumByBucket[] ): Promise { - return this.computePartialChecksumsForCollection( - batch, - this.db.v1_bucket_data as unknown as mongo.Collection, - (request) => ({ - _id: { - $gt: { - g: this.group_id, - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - g: this.group_id, - b: request.bucket, - o: request.end - } + return this.computePartialChecksumsForCollection(batch, this.db.v1_bucket_data, (request) => ({ + _id: { + $gt: { + g: this.group_id, + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + g: this.group_id, + b: request.bucket, + o: request.end } - }) - ); + } + })); } protected async fetchPreStates( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index 9dd0bb519..dbe219c5b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -15,6 +15,7 @@ import { MongoChecksumOptions } from '../common/MongoChecksumsBase.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; +import { BucketDataDocumentBase } from '../models.js'; export class MongoChecksumsV3Impl extends AbstractMongoChecksums { declare protected readonly db: VersionedPowerSyncMongoV3; @@ -50,7 +51,7 @@ export class MongoChecksumsV3Impl extends AbstractMongoChecksums { for (const [definitionId, requests] of requestsByDefinition.entries()) { const groupResults = await this.computePartialChecksumsForCollection( requests, - this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, + this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, createV3BucketFilter ); for (const checksum of groupResults.values()) { From b4ac135c2dfbb77b9a18eab3d2a11a3212a30b49 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 12:07:10 +0200 Subject: [PATCH 64/93] Type tweak. --- .../src/storage/implementation/v3/MongoChecksumsV3.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index dbe219c5b..98ea63889 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -51,7 +51,7 @@ export class MongoChecksumsV3Impl extends AbstractMongoChecksums { for (const [definitionId, requests] of requestsByDefinition.entries()) { const groupResults = await this.computePartialChecksumsForCollection( requests, - this.db.bucket_data_v3(this.group_id, definitionId) as unknown as mongo.Collection, + this.db.bucket_data_v3(this.group_id, definitionId), createV3BucketFilter ); for (const checksum of groupResults.values()) { From 6e46e4695efcce5ad93681dd30c85e2271a2f3fe Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 12:31:05 +0200 Subject: [PATCH 65/93] Simplify MongoChecksums. --- .../implementation/common/MongoChecksums.ts | 383 +++++++++++++++--- .../common/MongoChecksumsBase.ts | 366 ----------------- .../common/MongoSyncBucketStorageBase.ts | 8 +- .../implementation/v1/MongoChecksumsV1.ts | 7 +- .../implementation/v1/MongoCompactorV1.ts | 5 +- .../v1/MongoSyncBucketStorageV1.ts | 34 +- .../implementation/v3/MongoChecksumsV3.ts | 17 +- .../implementation/v3/MongoCompactorV3.ts | 9 +- .../v3/MongoSyncBucketStorageV3.ts | 12 + 9 files changed, 386 insertions(+), 455 deletions(-) delete mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts index 2bd48d45e..02dd6161a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts @@ -1,79 +1,356 @@ import { + addPartialChecksums, + bson, + BucketChecksum, BucketChecksumRequest, + ChecksumCache, ChecksumMap, FetchPartialBucketChecksum, InternalOpId, - PartialChecksumMap + isPartialChecksum, + PartialChecksum, + PartialChecksumMap, + PartialOrFullChecksum } from '@powersync/service-core'; -import { ServiceAssertionError } from '@powersync/lib-services-framework'; import type { VersionedPowerSyncMongo } from '../db.js'; -import { MongoChecksumsV1Impl } from '../v1/MongoChecksumsV1.js'; -import { MongoChecksumsV3Impl } from '../v3/MongoChecksumsV3.js'; -import type { VersionedPowerSyncMongoV3 } from '../v3/VersionedPowerSyncMongoV3.js'; -import { - AbstractMongoChecksums, - FetchPartialBucketChecksumByBucket, - FetchPartialBucketChecksumV3, - MongoChecksumOptions -} from './MongoChecksumsBase.js'; - -export { - FetchPartialBucketChecksumByBucket, - FetchPartialBucketChecksumV3, - MongoChecksumOptions -} from './MongoChecksumsBase.js'; + +import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { BucketDataDocumentBase, StorageConfig } from '../models.js'; + +export interface FetchPartialBucketChecksumV3 { + bucket: string; + definitionId: BucketDefinitionId; + start?: InternalOpId; + end: InternalOpId; +} + +export interface FetchPartialBucketChecksumByBucket { + bucket: string; + start?: InternalOpId; + end: InternalOpId; +} /** - * Public checksum API. Delegates to a storage-version-specific implementation. + * Checksum calculation options, primarily for tests. */ -export class MongoChecksums { - private readonly impl: AbstractMongoChecksums; - private readonly v3Impl: MongoChecksumsV3Impl | null; - private readonly v1Impl: MongoChecksumsV1Impl | null; - - constructor(db: VersionedPowerSyncMongo, group_id: number, options: MongoChecksumOptions) { - if (options.storageConfig.incrementalReprocessing) { - this.v3Impl = new MongoChecksumsV3Impl( - db as VersionedPowerSyncMongoV3, - group_id, - options, - options.mapping ?? - (() => { - throw new ServiceAssertionError('BucketDefinitionMapping is required for v3 MongoDB checksum queries'); - })() - ); - this.v1Impl = null; - this.impl = this.v3Impl; - } else { - this.v3Impl = null; - this.v1Impl = new MongoChecksumsV1Impl(db, group_id, options); - this.impl = this.v1Impl; - } +export interface MongoChecksumOptions { + /** + * How many buckets to process in a batch when calculating checksums. + */ + bucketBatchLimit?: number; + + /** + * Limit on the number of documents to calculate a checksum on at a time. + */ + operationBatchLimit?: number; + + storageConfig: StorageConfig; + mapping?: BucketDefinitionMapping; +} + +const DEFAULT_BUCKET_BATCH_LIMIT = 200; +const DEFAULT_OPERATION_BATCH_LIMIT = 50_000; + +export abstract class MongoChecksums { + private _cache: ChecksumCache | undefined; + private readonly storageConfig: StorageConfig; + + constructor( + protected readonly db: VersionedPowerSyncMongo, + protected readonly group_id: number, + protected readonly options: MongoChecksumOptions + ) { + this.storageConfig = options.storageConfig; + } + + /** + * Lazy-instantiated cache. + * + * This means the cache only allocates memory once it is used for the first time. + */ + private get cache(): ChecksumCache { + this._cache ??= new ChecksumCache({ + fetchChecksums: (batch) => { + return this.computePartialChecksums(batch); + } + }); + return this._cache; } + /** + * Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from + * the database (bucket_state + bucket_data). + */ async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise { - return this.impl.getChecksums(checkpoint, buckets); + return this.cache.getChecksumMap(checkpoint, buckets); } clearCache() { - this.impl.clearCache(); + this.cache.clear(); } - async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise { - return this.impl.computePartialChecksumsDirect(batch); + /** + * Calculate (partial) checksums from bucket_state (pre-aggregated) and bucket_data (individual operations). + * + * Results are not cached here. This method is only called by {@link ChecksumCache.getChecksumMap}, + * which is responsible for caching its result. + * + * As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state + * can be slow. + */ + private async computePartialChecksums(batch: FetchPartialBucketChecksum[]): Promise { + if (batch.length == 0) { + return new Map(); + } + const preStates = await this.fetchPreStates(batch); + + const mappedRequests = batch.map((request) => { + let start = request.start; + if (start == null) { + const preState = preStates.get(request.bucket); + if (preState != null) { + start = preState.opId; + } + } + return { + ...request, + start + }; + }); + + const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests); + + return new Map( + batch.map((request) => { + const bucket = request.bucket; + // Could be null if this is either (1) a partial request, or (2) no compacted checksum was available + const preState = preStates.get(bucket); + // Could be null if we got no data + const partialChecksum = queriedChecksums.get(bucket); + const merged = addPartialChecksums(bucket, preState?.checksum ?? null, partialChecksum ?? null); + + return [bucket, merged]; + }) + ); } - async computePartialChecksumsDirectV1(batch: FetchPartialBucketChecksumByBucket[]): Promise { - if (this.v1Impl == null) { - throw new ServiceAssertionError('V1 checksum routing is only available when incrementalReprocessing is disabled'); + /** + * Calculate (partial) checksums from the data collection directly, bypassing the cache and bucket_state. + * + * Can be used directly in cases where the cache should be bypassed, such as from a compact job. + * + * Internally, we do calculations in smaller batches of buckets as appropriate. + * + * For large buckets, this can be slow, but should not time out as the underlying queries are performed in + * smaller batches. + */ + public async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise { + // Limit the number of buckets we query for at a time. + const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT; + + if (batch.length <= bucketBatchLimit) { + // Single batch - no need for splitting the batch and merging results + return await this.computePartialChecksumsInternal(batch); } - return this.v1Impl.computePartialChecksumsDirectByBucket(batch); + // Split the batch and merge results + let results = new Map(); + for (let i = 0; i < batch.length; i += bucketBatchLimit) { + const bucketBatch = batch.slice(i, i + bucketBatchLimit); + const batchResults = await this.computePartialChecksumsInternal(bucketBatch); + for (let r of batchResults.values()) { + results.set(r.bucket, r); + } + } + return results; } - async computePartialChecksumsDirectV3(batch: FetchPartialBucketChecksumV3[]): Promise { - if (this.v3Impl == null) { - throw new ServiceAssertionError('V3 checksum routing is only available when incrementalReprocessing is enabled'); + /** + * Query a batch of checksums. + * + * We limit the number of operations that the query aggregates in each sub-batch, to avoid potential query timeouts. + * + * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this. + */ + protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise; + + protected abstract fetchPreStates( + batch: FetchPartialBucketChecksum[] + ): Promise>; + + protected async computePartialChecksumsForCollection< + TRequest extends FetchPartialBucketChecksumByBucket, + TBucketDataDocument extends BucketDataDocumentBase + >( + batch: TRequest[], + collection: lib_mongo.mongo.Collection, + createFilter: (request: TRequest) => any + ): Promise { + const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT; + + // Map requests by bucket. We adjust this as we get partial results. + let requests = new Map(); + for (let request of batch) { + requests.set(request.bucket, request); + } + + const partialChecksums = new Map(); + + while (requests.size > 0) { + const filters = Array.from(requests.values(), createFilter); + + // Historically, checksum may be stored as 'int' or 'double'. + // More recently, this should be a 'long'. + // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations. + const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' }; + + // Aggregate over a max of `batchLimit` operations at a time. + // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12. + // Then we'll do three batches: + // 1. Query: A[1-end], B[1-end], C[1-end] + // Returns: A[1-10], B[1-2] + // 2. Query: B[3-end], C[1-end] + // Returns: B[3-10], C[1-4] + // 3. Query: C[5-end] + // Returns: C[5-10] + const aggregate = await collection + .aggregate( + [ + { + $match: { + $or: filters + } + }, + // sort and limit _before_ grouping + { $sort: { _id: 1 } }, + { $limit: batchLimit }, + { + $group: { + _id: '$_id.b', + checksum_total: { $sum: checksumLong }, + count: { $sum: 1 }, + has_clear_op: { + $max: { + $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0] + } + }, + last_op: { $max: '$_id.o' } + } + }, + // Sort the aggregated results (100 max, so should be fast). + // This is important to identify which buckets we have partial data for. + { $sort: { _id: 1 } } + ], + { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS } + ) + .toArray() + .catch((e) => { + throw lib_mongo.mapQueryError(e, 'while reading checksums'); + }); + + let batchCount = 0; + let limitReached = false; + for (let doc of aggregate) { + const bucket = doc._id; + const checksum = checksumFromAggregate(doc); + + const existing = partialChecksums.get(bucket); + if (existing != null) { + partialChecksums.set(bucket, addPartialChecksums(bucket, existing, checksum)); + } else { + partialChecksums.set(bucket, checksum); + } + + batchCount += doc.count; + if (batchCount == batchLimit) { + // Limit reached. Request more in the next batch. + // Note that this only affects the _last_ bucket in a batch. + limitReached = true; + const req = requests.get(bucket); + requests.set(bucket, { + ...req!, + start: doc.last_op + }); + } else { + // All done for this bucket + requests.delete(bucket); + } + } + if (!limitReached) { + break; + } } - return this.v3Impl.computePartialChecksumsDirectByDefinition(batch); + + return new Map( + batch.map((request) => { + const bucket = request.bucket; + // Could be null if we got no data + let partialChecksum = partialChecksums.get(bucket); + if (partialChecksum == null) { + partialChecksum = { + bucket, + partialCount: 0, + partialChecksum: 0 + }; + } + if (request.start == null && isPartialChecksum(partialChecksum)) { + partialChecksum = { + bucket, + count: partialChecksum.partialCount, + checksum: partialChecksum.partialChecksum + }; + } + + return [bucket, partialChecksum]; + }) + ); + } +} + +export function createV3BucketFilter(request: Pick) { + return { + _id: { + $gt: { + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + b: request.bucket, + o: request.end + } + } + }; +} + +export function emptyChecksumForRequest( + request: Pick +): PartialOrFullChecksum { + if (request.start == null) { + return { bucket: request.bucket, count: 0, checksum: 0 }; + } + return { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }; +} + +/** + * Convert output of the $group stage into a checksum. + */ +function checksumFromAggregate(doc: bson.Document): PartialOrFullChecksum { + const partialChecksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff; + const bucket = doc._id; + + if (doc.has_clear_op == 1) { + return { + // full checksum - replaces any previous one + bucket, + checksum: partialChecksum, + count: doc.count + } satisfies BucketChecksum; + } else { + return { + // partial checksum - is added to a previous one + bucket, + partialCount: doc.count, + partialChecksum + } satisfies PartialChecksum; } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts deleted file mode 100644 index 20d139468..000000000 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksumsBase.ts +++ /dev/null @@ -1,366 +0,0 @@ -import * as lib_mongo from '@powersync/lib-service-mongodb'; -import { mongo } from '@powersync/lib-service-mongodb'; -import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { - addPartialChecksums, - bson, - BucketChecksumRequest, - BucketChecksum, - ChecksumCache, - ChecksumMap, - FetchPartialBucketChecksum, - InternalOpId, - isPartialChecksum, - PartialChecksum, - PartialChecksumMap, - PartialOrFullChecksum -} from '@powersync/service-core'; -import type { VersionedPowerSyncMongo } from '../db.js'; -import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; -import { BucketDataDocumentBase, StorageConfig } from '../models.js'; - -export interface FetchPartialBucketChecksumV3 { - bucket: string; - definitionId: BucketDefinitionId; - start?: InternalOpId; - end: InternalOpId; -} - -export interface FetchPartialBucketChecksumByBucket { - bucket: string; - start?: InternalOpId; - end: InternalOpId; -} - -/** - * Checksum calculation options, primarily for tests. - */ -export interface MongoChecksumOptions { - /** - * How many buckets to process in a batch when calculating checksums. - */ - bucketBatchLimit?: number; - - /** - * Limit on the number of documents to calculate a checksum on at a time. - */ - operationBatchLimit?: number; - - storageConfig: StorageConfig; - mapping?: BucketDefinitionMapping; -} - -const DEFAULT_BUCKET_BATCH_LIMIT = 200; -const DEFAULT_OPERATION_BATCH_LIMIT = 50_000; - -/** - * Shared checksum query plumbing. - * - * General implementation flow is: - * 1. getChecksums() -> check cache for (partial) matches. If not found or partial match, query the remainder using computePartialChecksums(). - * 2. computePartialChecksums() -> query bucket_state for partial matches. Query the remainder using computePartialChecksumsDirect(). - * 3. computePartialChecksumsDirect() -> split into batches of 200 buckets at a time -> computePartialChecksumsInternal() - * 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time - */ -export abstract class AbstractMongoChecksums { - private _cache: ChecksumCache | undefined; - private readonly storageConfig: StorageConfig; - - constructor( - protected readonly db: VersionedPowerSyncMongo, - protected readonly group_id: number, - protected readonly options: MongoChecksumOptions - ) { - this.storageConfig = options.storageConfig; - } - - /** - * Lazy-instantiated cache. - * - * This means the cache only allocates memory once it is used for the first time. - */ - private get cache(): ChecksumCache { - this._cache ??= new ChecksumCache({ - fetchChecksums: (batch) => { - return this.computePartialChecksums(batch); - } - }); - return this._cache; - } - - /** - * Calculate checksums, utilizing the cache for partial checkums, and querying the remainder from - * the database (bucket_state + bucket_data). - */ - async getChecksums(checkpoint: InternalOpId, buckets: BucketChecksumRequest[]): Promise { - return this.cache.getChecksumMap(checkpoint, buckets); - } - - clearCache() { - this.cache.clear(); - } - - /** - * Calculate (partial) checksums from bucket_state (pre-aggregated) and bucket_data (individual operations). - * - * Results are not cached here. This method is only called by {@link ChecksumCache.getChecksumMap}, - * which is responsible for caching its result. - * - * As long as data is compacted regularly, this should be fast. Large buckets without pre-compacted bucket_state - * can be slow. - */ - private async computePartialChecksums(batch: FetchPartialBucketChecksum[]): Promise { - if (batch.length == 0) { - return new Map(); - } - const preStates = await this.fetchPreStates(batch); - - const mappedRequests = batch.map((request) => { - let start = request.start; - if (start == null) { - const preState = preStates.get(request.bucket); - if (preState != null) { - start = preState.opId; - } - } - return { - ...request, - start - }; - }); - - const queriedChecksums = await this.computePartialChecksumsDirect(mappedRequests); - - return new Map( - batch.map((request) => { - const bucket = request.bucket; - // Could be null if this is either (1) a partial request, or (2) no compacted checksum was available - const preState = preStates.get(bucket); - // Could be null if we got no data - const partialChecksum = queriedChecksums.get(bucket); - const merged = addPartialChecksums(bucket, preState?.checksum ?? null, partialChecksum ?? null); - - return [bucket, merged]; - }) - ); - } - - /** - * Calculate (partial) checksums from the data collection directly, bypassing the cache and bucket_state. - * - * Can be used directly in cases where the cache should be bypassed, such as from a compact job. - * - * Internally, we do calculations in smaller batches of buckets as appropriate. - * - * For large buckets, this can be slow, but should not time out as the underlying queries are performed in - * smaller batches. - */ - public async computePartialChecksumsDirect(batch: FetchPartialBucketChecksum[]): Promise { - // Limit the number of buckets we query for at a time. - const bucketBatchLimit = this.options?.bucketBatchLimit ?? DEFAULT_BUCKET_BATCH_LIMIT; - - if (batch.length <= bucketBatchLimit) { - // Single batch - no need for splitting the batch and merging results - return await this.computePartialChecksumsInternal(batch); - } - // Split the batch and merge results - let results = new Map(); - for (let i = 0; i < batch.length; i += bucketBatchLimit) { - const bucketBatch = batch.slice(i, i + bucketBatchLimit); - const batchResults = await this.computePartialChecksumsInternal(bucketBatch); - for (let r of batchResults.values()) { - results.set(r.bucket, r); - } - } - return results; - } - - /** - * Query a batch of checksums. - * - * We limit the number of operations that the query aggregates in each sub-batch, to avoid potential query timeouts. - * - * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this. - */ - protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise; - - protected abstract fetchPreStates( - batch: FetchPartialBucketChecksum[] - ): Promise>; - - protected async computePartialChecksumsForCollection< - TRequest extends FetchPartialBucketChecksumByBucket, - TBucketDataDocument extends BucketDataDocumentBase - >( - batch: TRequest[], - collection: mongo.Collection, - createFilter: (request: TRequest) => any - ): Promise { - const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT; - - // Map requests by bucket. We adjust this as we get partial results. - let requests = new Map(); - for (let request of batch) { - requests.set(request.bucket, request); - } - - const partialChecksums = new Map(); - - while (requests.size > 0) { - const filters = Array.from(requests.values(), createFilter); - - // Historically, checksum may be stored as 'int' or 'double'. - // More recently, this should be a 'long'. - // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations. - const checksumLong = this.storageConfig.longChecksums ? '$checksum' : { $toLong: '$checksum' }; - - // Aggregate over a max of `batchLimit` operations at a time. - // Let's say we have 3 buckets (A, B, C), each with 10 operations, and our batch limit is 12. - // Then we'll do three batches: - // 1. Query: A[1-end], B[1-end], C[1-end] - // Returns: A[1-10], B[1-2] - // 2. Query: B[3-end], C[1-end] - // Returns: B[3-10], C[1-4] - // 3. Query: C[5-end] - // Returns: C[5-10] - const aggregate = await collection - .aggregate( - [ - { - $match: { - $or: filters - } - }, - // sort and limit _before_ grouping - { $sort: { _id: 1 } }, - { $limit: batchLimit }, - { - $group: { - _id: '$_id.b', - checksum_total: { $sum: checksumLong }, - count: { $sum: 1 }, - has_clear_op: { - $max: { - $cond: [{ $eq: ['$op', 'CLEAR'] }, 1, 0] - } - }, - last_op: { $max: '$_id.o' } - } - }, - // Sort the aggregated results (100 max, so should be fast). - // This is important to identify which buckets we have partial data for. - { $sort: { _id: 1 } } - ], - { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS } - ) - .toArray() - .catch((e) => { - throw lib_mongo.mapQueryError(e, 'while reading checksums'); - }); - - let batchCount = 0; - let limitReached = false; - for (let doc of aggregate) { - const bucket = doc._id; - const checksum = checksumFromAggregate(doc); - - const existing = partialChecksums.get(bucket); - if (existing != null) { - partialChecksums.set(bucket, addPartialChecksums(bucket, existing, checksum)); - } else { - partialChecksums.set(bucket, checksum); - } - - batchCount += doc.count; - if (batchCount == batchLimit) { - // Limit reached. Request more in the next batch. - // Note that this only affects the _last_ bucket in a batch. - limitReached = true; - const req = requests.get(bucket); - requests.set(bucket, { - ...req!, - start: doc.last_op - }); - } else { - // All done for this bucket - requests.delete(bucket); - } - } - if (!limitReached) { - break; - } - } - - return new Map( - batch.map((request) => { - const bucket = request.bucket; - // Could be null if we got no data - let partialChecksum = partialChecksums.get(bucket); - if (partialChecksum == null) { - partialChecksum = { - bucket, - partialCount: 0, - partialChecksum: 0 - }; - } - if (request.start == null && isPartialChecksum(partialChecksum)) { - partialChecksum = { - bucket, - count: partialChecksum.partialCount, - checksum: partialChecksum.partialChecksum - }; - } - - return [bucket, partialChecksum]; - }) - ); - } -} - -export function createV3BucketFilter(request: Pick) { - return { - _id: { - $gt: { - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - b: request.bucket, - o: request.end - } - } - }; -} - -export function emptyChecksumForRequest( - request: Pick -): PartialOrFullChecksum { - if (request.start == null) { - return { bucket: request.bucket, count: 0, checksum: 0 }; - } - return { bucket: request.bucket, partialCount: 0, partialChecksum: 0 }; -} - -/** - * Convert output of the $group stage into a checksum. - */ -function checksumFromAggregate(doc: bson.Document): PartialOrFullChecksum { - const partialChecksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff; - const bucket = doc._id; - - if (doc.has_clear_op == 1) { - return { - // full checksum - replaces any previous one - bucket, - checksum: partialChecksum, - count: doc.count - } satisfies BucketChecksum; - } else { - return { - // partial checksum - is added to a previous one - bucket, - partialCount: doc.count, - partialChecksum - } satisfies PartialChecksum; - } -} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts index 3c9c29380..83bd721e2 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts @@ -85,11 +85,7 @@ export abstract class BaseMongoSyncBucketStorage ) { super(); this.db = factory.db.versioned(sync_rules.getStorageConfig()); - this.checksums = new MongoChecksums(this.db, this.group_id, { - ...options.checksumOptions, - storageConfig: options?.storageConfig, - mapping: sync_rules.mapping - }); + this.checksums = this.createMongoChecksums(options); this.writeCheckpointAPI = new MongoWriteCheckpointAPI({ db: this.db, mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED, @@ -97,6 +93,8 @@ export abstract class BaseMongoSyncBucketStorage }); } + protected abstract createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums; + get writeCheckpointMode() { return this.writeCheckpointAPI.writeCheckpointMode; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts index cea25a5a1..c39955ffe 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts @@ -1,4 +1,3 @@ -import { mongo } from '@powersync/lib-service-mongodb'; import { bson, BucketChecksum, @@ -6,11 +5,11 @@ import { InternalOpId, PartialChecksumMap } from '@powersync/service-core'; -import { AbstractMongoChecksums, FetchPartialBucketChecksumByBucket } from '../common/MongoChecksumsBase.js'; +import { FetchPartialBucketChecksumByBucket } from '../common/MongoChecksums.js'; +import { MongoChecksums } from '../MongoChecksums.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; -import { BucketDataDocumentBase } from '../models.js'; -export class MongoChecksumsV1Impl extends AbstractMongoChecksums { +export class MongoChecksumsV1 extends MongoChecksums { declare protected readonly db: VersionedPowerSyncMongoV1; async computePartialChecksumsDirectByBucket( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index f933813fd..cf5131d6d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -14,9 +14,12 @@ import { import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { BaseMongoCompactor, DirtyBucket } from '../common/MongoCompactorBase.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; +import { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; export class MongoCompactorV1 extends BaseMongoCompactor { + // Override types to the more specific ones declare protected readonly db: VersionedPowerSyncMongoV1; + declare protected readonly storage: MongoSyncBucketStorageV1; public async *dirtyBucketBatches(options: { minBucketChanges: number; @@ -63,7 +66,7 @@ export class MongoCompactorV1 extends BaseMongoCompactor { protected async computeChecksumsForBuckets( buckets: Pick[] ): Promise { - return this.storage.checksums.computePartialChecksumsDirectV1( + return this.storage.checksums.computePartialChecksumsDirectByBucket( buckets.map(({ bucket }) => ({ bucket, end: this.maxOpId diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 35a134762..85459216b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -10,31 +10,35 @@ import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { JSONBig } from '@powersync/service-jsonbig'; import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; +import { MongoBucketStorage } from '../../MongoBucketStorage.js'; +import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; +import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; +import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorageBase.js'; import { + MongoSyncBucketStorageCheckpoint, + MongoSyncBucketStorageContext +} from '../common/MongoSyncBucketStorageContext.js'; +import { + bucketDataDocumentToTagged, BucketDataDocumentV1, BucketDataKeyV1, BucketStateDocument, CommonSourceTableDocument, - LEGACY_BUCKET_DATA_DEFINITION_ID, - bucketDataDocumentToTagged + LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js'; -import { - MongoSyncBucketStorageCheckpoint, - MongoSyncBucketStorageContext -} from '../common/MongoSyncBucketStorageContext.js'; -import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorageBase.js'; -import { MongoBucketStorage } from '../../MongoBucketStorage.js'; -import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; -import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; +import { MongoChecksumsV1 } from './MongoChecksumsV1.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { + // Declare types to be more specific declare readonly db: VersionedPowerSyncMongoV1; + declare readonly checksums: MongoChecksumsV1; constructor( factory: MongoBucketStorage, @@ -53,6 +57,14 @@ export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { return new MongoBucketBatchV1(batchOptions); } + protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums { + return new MongoChecksumsV1(this.db, this.group_id, { + ...options.checksumOptions, + storageConfig: options?.storageConfig, + mapping: this.sync_rules.mapping + }); + } + protected sourceTableBaseId(): Partial { return { group_id: this.group_id }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index 98ea63889..6597be833 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -1,4 +1,3 @@ -import { mongo } from '@powersync/lib-service-mongodb'; import { BucketChecksum, FetchPartialBucketChecksum, @@ -8,25 +7,21 @@ import { } from '@powersync/service-core'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { - AbstractMongoChecksums, createV3BucketFilter, emptyChecksumForRequest, FetchPartialBucketChecksumV3, MongoChecksumOptions -} from '../common/MongoChecksumsBase.js'; +} from '../common/MongoChecksums.js'; +import { MongoChecksums } from '../MongoChecksums.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; -import { BucketDataDocumentBase } from '../models.js'; -export class MongoChecksumsV3Impl extends AbstractMongoChecksums { +export class MongoChecksumsV3 extends MongoChecksums { declare protected readonly db: VersionedPowerSyncMongoV3; + private readonly mapping: BucketDefinitionMapping; - constructor( - db: VersionedPowerSyncMongoV3, - group_id: number, - options: MongoChecksumOptions, - private readonly mapping: BucketDefinitionMapping - ) { + constructor(db: VersionedPowerSyncMongoV3, group_id: number, options: MongoChecksumOptions) { super(db, group_id, options); + this.mapping = options.mapping!; } private normalizeBatch(batch: FetchPartialBucketChecksum[]): FetchPartialBucketChecksumV3[] { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index ec9a5bd24..6dad4644f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -1,21 +1,22 @@ import { MONGO_OPERATION_TIMEOUT_MS, mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { BaseMongoCompactor, DirtyBucket } from '../common/MongoCompactorBase.js'; import { BucketDataDocumentBase, - BucketDataDocumentV3, BucketDataKeyV3, BucketStateDocumentBase, BucketStateDocumentV3, TaggedBucketDataDocument, taggedBucketDataDocumentToV3 } from '../models.js'; -import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { BaseMongoCompactor, DirtyBucket } from '../common/MongoCompactorBase.js'; +import { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoCompactorV3 extends BaseMongoCompactor { declare protected readonly db: VersionedPowerSyncMongoV3; + declare protected readonly storage: MongoSyncBucketStorageV3; public async *dirtyBucketBatches(options: { minBucketChanges: number; @@ -56,7 +57,7 @@ export class MongoCompactorV3 extends BaseMongoCompactor { protected async computeChecksumsForBuckets( buckets: Pick[] ): Promise { - return this.storage.checksums.computePartialChecksumsDirectV3( + return this.storage.checksums.computePartialChecksumsDirectByDefinition( buckets.map(({ bucket, definitionId }) => { if (definitionId == null) { throw new ServiceAssertionError(`Missing definitionId for V3 bucket checksum update on bucket ${bucket}`); diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 275999b52..b591fb5d9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -26,9 +26,13 @@ import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; import { CommonSourceTableDocument } from '../models.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; +import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; +import { MongoChecksums } from '../MongoChecksums.js'; export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { + // Declare types to be more specific declare readonly db: VersionedPowerSyncMongoV3; + declare readonly checksums: MongoChecksumsV3; constructor( factory: MongoBucketStorage, @@ -68,6 +72,14 @@ export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { } } + protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums { + return new MongoChecksumsV3(this.db, this.group_id, { + ...options.checksumOptions, + storageConfig: options?.storageConfig, + mapping: this.sync_rules.mapping + }); + } + protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch { return new MongoBucketBatchV3(batchOptions); } From f74137b97674c5f39110510e8c7e03e390b00dc3 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 12:47:34 +0200 Subject: [PATCH 66/93] Remove MongoCompactorBase. --- .../implementation/common/MongoCompactor.ts | 760 +++++++++++++++++- .../common/MongoCompactorBase.ts | 756 ----------------- .../common/MongoSyncBucketStorageBase.ts | 7 +- .../implementation/v1/MongoCompactorV1.ts | 4 +- .../v1/MongoSyncBucketStorageV1.ts | 6 + .../implementation/v3/MongoCompactorV3.ts | 4 +- .../v3/MongoSyncBucketStorageV3.ts | 6 + .../test/src/storage_compacting.test.ts | 12 +- 8 files changed, 768 insertions(+), 787 deletions(-) delete mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts index 3716ce9ac..710b58a9e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts @@ -1,36 +1,756 @@ -import { PopulateChecksumCacheResults } from '@powersync/service-core'; +import { isMongoServerError, mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb'; +import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { + addChecksums, + InternalOpId, + isPartialChecksum, + PopulateChecksumCacheResults, + storage, + utils +} from '@powersync/service-core'; + import type { VersionedPowerSyncMongo } from '../db.js'; -import type { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; -import { MongoCompactorV1 } from '../v1/MongoCompactorV1.js'; -import { MongoCompactorV3 } from '../v3/MongoCompactorV3.js'; -import { BaseMongoCompactor, DirtyBucket, MongoCompactOptions } from './MongoCompactorBase.js'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { + BucketDataDocumentBase, + BucketDataDocumentV1, + BucketDataDocumentV3, + LEGACY_BUCKET_DATA_DEFINITION_ID, + TaggedBucketDataDocument, + BucketStateDocumentBase, + bucketDataDocumentToTagged +} from '../models.js'; +import { cacheKey } from '../OperationBatch.js'; +import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; -export { DirtyBucket, MongoCompactOptions } from './MongoCompactorBase.js'; +interface CurrentBucketState { + /** Bucket name */ + bucket: string; + definitionId: BucketDefinitionId; + /** + * Rows seen in the bucket, with the last op_id of each. + */ + seen: Map; + /** + * Estimated memory usage of the seen Map. + */ + trackingSize: number; + /** + * Last (lowest) seen op_id that is not a PUT. + */ + lastNotPut: InternalOpId | null; + /** + * Number of REMOVE/MOVE operations seen since lastNotPut. + */ + opsSincePut: number; + /** + * Incrementally-updated checksum, up to maxOpId. + */ + checksum: number; + /** + * Op count for the checksum. + */ + opCount: number; + /** + * Byte size of ops covered by the checksum. + */ + opBytes: number; +} -export class MongoCompactor { - private readonly impl: BaseMongoCompactor; +type CompactBucketDataDocument = Pick< + TaggedBucketDataDocument, + '_id' | 'def' | 'op' | 'table' | 'row_id' | 'source_table' | 'source_key' | 'checksum' | 'target_op' +> & { + size: number | bigint; +}; - constructor(storage: MongoSyncBucketStorage, db: VersionedPowerSyncMongo, options: MongoCompactOptions) { - if (db.storageConfig.incrementalReprocessing) { - this.impl = new MongoCompactorV3(storage, db, options); - } else { - this.impl = new MongoCompactorV1(storage, db, options); - } +type CompactClearBucketDataDocument = Pick; +type BucketDataCollectionDocument = BucketDataDocumentV1 | BucketDataDocumentV3; +type BucketDataClearProjection = { + _id: BucketDataDocumentBase['_id']; + op: CompactClearBucketDataDocument['op']; + checksum: bigint; + target_op?: bigint | null; +}; + +export interface MongoCompactOptions extends storage.CompactOptions {} + +const DEFAULT_CLEAR_BATCH_LIMIT = 5000; +const DEFAULT_MOVE_BATCH_LIMIT = 2000; +const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; +const DEFAULT_MIN_BUCKET_CHANGES = 10; +const DEFAULT_MIN_CHANGE_RATIO = 0.1; +const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; +/** This default is primarily for tests. */ +const DEFAULT_MEMORY_LIMIT_MB = 64; + +export interface DirtyBucket { + bucket: string; + definitionId: BucketDefinitionId | null; + estimatedCount: number; + dirtyRatio?: number; +} + +export abstract class MongoCompactor { + protected updates: mongo.AnyBulkWriteOperation[] = []; + protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; + protected activeBucketDataCollection: mongo.Collection | null = null; + protected activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; + + protected readonly idLimitBytes: number; + protected readonly moveBatchLimit: number; + protected readonly moveBatchQueryLimit: number; + protected readonly clearBatchLimit: number; + protected readonly minBucketChanges: number; + protected readonly minChangeRatio: number; + protected readonly maxOpId: bigint; + protected readonly buckets: string[] | undefined; + protected readonly signal?: AbortSignal; + protected readonly group_id: number; + + constructor( + protected readonly storage: MongoSyncBucketStorage, + protected readonly db: VersionedPowerSyncMongo, + options: MongoCompactOptions + ) { + this.group_id = storage.group_id; + this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; + this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; + this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; + this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; + this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; + this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; + this.maxOpId = options.maxOpId ?? 0n; + this.buckets = options.compactBuckets; + this.signal = options.signal; } + /** + * Compact buckets by converting operations into MOVE and/or CLEAR operations. + * + * See /docs/compacting-operations.md for details. + */ async compact() { - return this.impl.compact(); + if (this.buckets) { + for (const bucket of this.buckets) { + // We can make this more efficient later on by iterating through the buckets in a single query. + // That makes batching more tricky, so we leave for later. + await this.compactSingleBucketRetried(bucket); + } + } else { + await this.compactDirtyBuckets(); + } } + /** + * Subset of compact, only populating checksums where relevant. + */ async populateChecksums(options: { minBucketChanges: number }): Promise { - return this.impl.populateChecksums(options); + let count = 0; + while (true) { + this.signal?.throwIfAborted(); + const buckets = await this.dirtyBucketBatchForChecksums(options); + if (buckets.length == 0) { + break; + } + this.signal?.throwIfAborted(); + + const start = Date.now(); + // Filter batch by estimated bucket size, to reduce possibility of timeouts. + const checkBuckets: typeof buckets = []; + let totalCountEstimate = 0; + for (const bucket of buckets) { + checkBuckets.push(bucket); + totalCountEstimate += bucket.estimatedCount; + if (totalCountEstimate > 50_000) { + break; + } + } + logger.info( + `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` + ); + await this.updateChecksumsBatch(checkBuckets); + logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); + count += checkBuckets.length; + } + return { buckets: count }; + } + + protected async *dirtyBucketBatchesForCollection( + collection: mongo.Collection, + lastId: TCollectionBucketState['_id'], + maxId: TCollectionBucketState['_id'], + options: { + minBucketChanges: number; + minChangeRatio: number; + }, + getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null + ): AsyncGenerator { + while (true) { + // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline + // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria. + const [result] = await collection + .aggregate<{ + buckets: TCollectionBucketState[]; + cursor: Pick[]; + }>( + [ + { + $match: { + _id: { $gt: lastId, $lt: maxId } + } + }, + { + $sort: { _id: 1 } + }, + { + // Scan a fixed number of docs each query so sparse matches don't block progress. + $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE + }, + { + $facet: { + buckets: [ + { + $match: { + 'estimate_since_compact.count': { $gte: options.minBucketChanges } + } + }, + { + $project: { + _id: 1, + estimate_since_compact: 1, + compacted_state: 1 + } + } + ], + // This is used for the next query. + cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] + } + } + ], + { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } + ) + .toArray(); + + const cursor = result?.cursor?.[0]; + if (cursor == null) { + break; + } + lastId = cursor._id; + + const mapped = (result?.buckets ?? []).map((bucketState) => { + // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios. + // BigInt precision is not needed here since this is only an estimate. + const updatedCount = bucketState.estimate_since_compact?.count ?? 0; + const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; + const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); + const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; + const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; + const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; + return { + bucket: bucketState._id.b, + definitionId: getDefinitionId(bucketState), + estimatedCount: totalCount, + dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) + }; + }); + + yield mapped.filter( + (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio + ); + } + } + + protected async dirtyBucketBatchForChecksumsForCollection( + collection: mongo.Collection, + filter: mongo.Filter, + getDefinitionId: (state: mongo.WithId) => BucketDefinitionId | null + ): Promise { + const dirtyBuckets = await collection + .find(filter, { + projection: { + _id: 1, + estimate_since_compact: 1, + compacted_state: 1 + }, + sort: { + 'estimate_since_compact.count': -1 + }, + limit: 200, + maxTimeMS: MONGO_OPERATION_TIMEOUT_MS + }) + .toArray(); + + return dirtyBuckets.map((bucket) => ({ + bucket: bucket._id.b, + definitionId: getDefinitionId(bucket), + estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) + })); + } + + public abstract dirtyBucketBatches(options: { + minBucketChanges: number; + minChangeRatio: number; + }): AsyncGenerator; + + public abstract dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise; + + protected async compactDirtyBuckets() { + for await (const buckets of this.dirtyBucketBatches({ + minBucketChanges: this.minBucketChanges, + minChangeRatio: this.minChangeRatio + })) { + this.signal?.throwIfAborted(); + if (buckets.length == 0) { + continue; + } + + for (const { bucket, definitionId } of buckets) { + await this.compactSingleBucketRetried(bucket, definitionId); + } + } + } + + /** + * Compaction for a single bucket, with retries on failure. + * + * This covers against occasional network or other database errors during a long compact job. + */ + protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { + let retryCount = 0; + while (true) { + try { + await this.compactSingleBucket(bucket, definitionId); + break; + } catch (e) { + if (retryCount < 3 && isMongoServerError(e)) { + logger.warn(`Error compacting bucket ${bucket}, retrying...`, e); + retryCount++; + await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount)); + } else { + throw e; + } + } + } + } + + protected async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { + const idLimitBytes = this.idLimitBytes; + const bucketCollection = await this.getBucketDataCollection(bucket, definitionId); + if (bucketCollection == null) { + return; + } + this.activeBucketDataCollection = bucketCollection.collection; + this.activeBucketDefinitionId = bucketCollection.definitionId; + try { + const currentState: CurrentBucketState = { + bucket, + definitionId: bucketCollection.definitionId, + seen: new Map(), + trackingSize: 0, + lastNotPut: null, + opsSincePut: 0, + checksum: 0, + opCount: 0, + opBytes: 0 + }; + + // Constant lower bound. + const lowerBound = this.bucketDataKey(bucket, new mongo.MinKey() as any); + // Upper bound is adjusted for each batch. + let upperBound = this.bucketDataKey(bucket, new mongo.MaxKey() as any); + + while (true) { + this.signal?.throwIfAborted(); + + // Query one batch at a time, to avoid cursor timeouts. + const pipeline = [ + { + $match: { + _id: { + $gte: lowerBound, + $lt: upperBound + }, + // Workaround for a clustered collection bug where the $lt operator may include upperBound. + // https://jira.mongodb.org/browse/SERVER-121822 + '_id.o': { $lt: upperBound.o } + } + }, + { $sort: { _id: -1 } }, + { $limit: this.moveBatchQueryLimit }, + { + $project: { + _id: 1, + op: 1, + table: 1, + row_id: 1, + source_table: 1, + source_key: 1, + checksum: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ]; + + const cursor = bucketCollection.collection.aggregate( + pipeline, + { + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: this.moveBatchQueryLimit + 1 + } + ); + // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. + // Instead, we load up to the limit. + const rawBatch = await cursor.toArray(); + const batch = rawBatch.map((document) => this.tagBucketDataDocument(document, bucketCollection.definitionId)); + + if (batch.length == 0) { + // We've reached the end. + break; + } + + // Reuse the exact collection _id value from Mongo for the next bound. + upperBound = rawBatch[rawBatch.length - 1]._id; + + for (const doc of batch) { + if (doc._id.o > this.maxOpId) { + continue; + } + + currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum)); + currentState.opCount += 1; + + let isPersistentPut = doc.op == 'PUT'; + + currentState.opBytes += Number(doc.size); + if (doc.op == 'REMOVE' || doc.op == 'PUT') { + const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; + const targetOp = currentState.seen.get(key); + if (targetOp) { + // Will convert to MOVE, so don't count as PUT. + isPersistentPut = false; + + this.updates.push({ + updateOne: { + filter: { _id: this.bucketDataKey(doc._id.b, doc._id.o) }, + update: { + $set: { + op: 'MOVE', + target_op: targetOp + }, + $unset: { + source_table: 1, + source_key: 1, + table: 1, + row_id: 1, + data: 1 + } + } satisfies mongo.UpdateFilter + } + }); + + // TODO: better estimate for this. + currentState.opBytes += 200 - Number(doc.size); + } else if (currentState.trackingSize < idLimitBytes) { + // flatstr reduces the memory usage by flattening the string. + currentState.seen.set(utils.flatstr(key), doc._id.o); + // length + 16 for the string + // 24 for the bigint + // 50 for map overhead + // 50 for additional overhead + currentState.trackingSize += key.length + 140; + } + } + + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (doc.op != 'CLEAR') { + if (currentState.lastNotPut == null) { + currentState.lastNotPut = doc._id.o; + } + currentState.opsSincePut += 1; + } + + if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { + await this.flush(); + } + } + + logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); + } + + // Free memory before clearing the bucket. + currentState.seen.clear(); + if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { + logger.info( + `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + // Need flush() before clear(). + await this.flush(); + await this.clearBucket(currentState); + } + + // Do this after clearBucket so we have accurate counts. + this.updateBucketChecksums(currentState); + // Need another flush after updateBucketChecksums(). + await this.flush(); + } finally { + this.activeBucketDataCollection = null; + this.activeBucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; + } + } + + protected updateBucketChecksums(state: CurrentBucketState) { + if (state.opCount < 0) { + throw new ServiceAssertionError( + `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` + ); + } + this.bucketStateUpdates.push({ + updateOne: { + filter: this.bucketStateFilter(state.bucket, state.definitionId), + update: { + $set: { + compacted_state: { + op_id: this.maxOpId, + count: state.opCount, + checksum: BigInt(state.checksum), + bytes: state.opBytes + }, + estimate_since_compact: { + // There could have been a whole bunch of new operations added to the bucket while compacting, + // which we don't currently cater for. We could potentially query for that, but that adds overhead. + count: 0, + bytes: 0 + } + } + } satisfies mongo.UpdateFilter, + // We generally expect this to have been created before. + // We don't create new ones here, to avoid issues with the unique index on bucket_updates. + upsert: false + } + }); + } + + protected async flush() { + if (this.updates.length > 0) { + logger.info(`Compacting ${this.updates.length} ops`); + if (this.activeBucketDataCollection == null) { + throw new ServiceAssertionError('No bucket_data collection selected for compaction'); + } + await this.activeBucketDataCollection.bulkWrite(this.updates, { + // Order is not important. Since checksums are not affected, these operations can happen in any order, + // and it's fine if the operations are partially applied. Each individual operation is atomic. + ordered: false + }); + this.updates = []; + } + if (this.bucketStateUpdates.length > 0) { + logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); + await this.flushBucketStateUpdates(); + this.bucketStateUpdates = []; + } + } + + /** + * Perform a CLEAR compact for a bucket. + * + * @param currentState tracks the last non-PUT op, which will be converted to CLEAR. + */ + protected async clearBucket(currentState: CurrentBucketState) { + const bucket = currentState.bucket; + const clearOp = currentState.lastNotPut!; + const bucketCollection = this.activeBucketDataCollection; + if (bucketCollection == null) { + throw new ServiceAssertionError('No bucket_data collection selected for compaction'); + } + + const opFilter = { + _id: { + $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), + $lte: this.bucketDataKey(bucket, clearOp) + } + }; + + const session = this.db.client.startSession(); + try { + let done = false; + while (!done) { + this.signal?.throwIfAborted(); + let opCountDiff = 0; + // Do the CLEAR operation in batches, with each batch a separate transaction. + // The state after each batch is fully consistent. + // We need a transaction per batch to make sure checksums stay consistent. + await session.withTransaction( + async () => { + const query = bucketCollection.find(opFilter as any, { + session, + sort: { _id: 1 }, + projection: { + _id: 1, + op: 1, + checksum: 1, + target_op: 1 + }, + limit: this.clearBatchLimit + }); + let checksum = 0; + let lastOp: CompactClearBucketDataDocument | null = null; + let targetOp: bigint | null = null; + let gotAnOp = false; + let numberOfOpsToClear = 0; + for await (const rawOp of query.stream()) { + const op = this.tagClearBucketDataDocument( + rawOp as BucketDataClearProjection, + this.activeBucketDefinitionId + ); + + if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') { + checksum = utils.addChecksums(checksum, Number(op.checksum)); + lastOp = op; + numberOfOpsToClear += 1; + if (op.op != 'CLEAR') { + gotAnOp = true; + } + if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) { + targetOp = op.target_op; + } + } else { + throw new ReplicationAssertionError( + `Unexpected ${op.op} operation at ${this.formatBucketDataKey(op._id)}` + ); + } + } + if (!gotAnOp) { + done = true; + return; + } + + logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?._id.o}`); + await bucketCollection.deleteMany( + { + _id: { + $gte: this.bucketDataKey(bucket, new mongo.MinKey()), + $lte: this.bucketDataKey(lastOp!._id.b, lastOp!._id.o) + } + }, + { session } + ); + + await bucketCollection.insertOne( + this.collectionBucketDataDocument({ + def: this.activeBucketDefinitionId, + _id: lastOp!._id, + op: 'CLEAR', + checksum: BigInt(checksum), + data: null, + target_op: targetOp + }), + { session } + ); + + opCountDiff = -numberOfOpsToClear + 1; + }, + { + writeConcern: { w: 'majority' }, + readConcern: { level: 'snapshot' } + } + ); + // Update outside the transaction, since the transaction can be retried multiple times. + currentState.opCount += opCountDiff; + } + } finally { + await session.endSession(); + } + } + + protected async updateChecksumsBatch(buckets: Pick[]) { + const checksums = await this.computeChecksumsForBuckets(buckets); + const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); + + for (const bucketChecksum of checksums.values()) { + if (isPartialChecksum(bucketChecksum)) { + // Should never happen since we don't specify `start`. + throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); + } + + this.bucketStateUpdates.push({ + updateOne: { + filter: this.bucketStateFilter( + bucketChecksum.bucket, + definitionIdByBucket.get(bucketChecksum.bucket) ?? null + ), + update: { + $set: { + compacted_state: { + op_id: this.maxOpId, + count: bucketChecksum.count, + checksum: BigInt(bucketChecksum.checksum), + bytes: null + }, + estimate_since_compact: { + count: 0, + bytes: 0 + } + } + } satisfies mongo.UpdateFilter, + // We don't create new ones here - it gets tricky to get the last_op right with the unique index on + // bucket_updates. + upsert: false + } + }); + } + + await this.flush(); } - dirtyBucketBatches(options: { minBucketChanges: number; minChangeRatio: number }): AsyncGenerator { - return this.impl.dirtyBucketBatches(options); + protected tagBucketDataDocument( + document: BucketDataCollectionDocument & { size: number | bigint }, + definitionId: BucketDefinitionId + ): CompactBucketDataDocument { + const tagged = bucketDataDocumentToTagged(document, definitionId); + return { + ...tagged, + size: document.size + }; } - dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise { - return this.impl.dirtyBucketBatchForChecksums(options); + protected tagClearBucketDataDocument( + document: BucketDataClearProjection, + definitionId: BucketDefinitionId + ): CompactClearBucketDataDocument { + return { + def: definitionId, + _id: { + b: document._id.b, + o: document._id.o + }, + op: document.op, + checksum: document.checksum, + target_op: document.target_op + }; } + + protected formatBucketDataKey(key: BucketDataDocumentBase['_id'] | { _id: BucketDataDocumentBase['_id'] }) { + const bucket = 'b' in key ? key.b : key._id.b; + const op = 'o' in key ? key.o : key._id.o; + return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; + } + + protected abstract flushBucketStateUpdates(): Promise; + protected abstract computeChecksumsForBuckets( + buckets: Pick[] + ): Promise; + protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; + protected abstract bucketDataKey( + bucket: string, + opId: InternalOpId | mongo.MinKey | mongo.MaxKey + ): BucketDataDocumentBase['_id']; + protected abstract getBucketDataCollection( + bucket: string, + definitionId: BucketDefinitionId | null + ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null>; + protected abstract collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts deleted file mode 100644 index 70825d5d5..000000000 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactorBase.ts +++ /dev/null @@ -1,756 +0,0 @@ -import { isMongoServerError, mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb'; -import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; -import { - addChecksums, - InternalOpId, - isPartialChecksum, - PopulateChecksumCacheResults, - storage, - utils -} from '@powersync/service-core'; - -import type { VersionedPowerSyncMongo } from '../db.js'; -import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { - BucketDataDocumentBase, - BucketDataDocumentV1, - BucketDataDocumentV3, - LEGACY_BUCKET_DATA_DEFINITION_ID, - TaggedBucketDataDocument, - BucketStateDocumentBase, - bucketDataDocumentToTagged -} from '../models.js'; -import type { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; -import { cacheKey } from '../OperationBatch.js'; - -interface CurrentBucketState { - /** Bucket name */ - bucket: string; - definitionId: BucketDefinitionId; - /** - * Rows seen in the bucket, with the last op_id of each. - */ - seen: Map; - /** - * Estimated memory usage of the seen Map. - */ - trackingSize: number; - /** - * Last (lowest) seen op_id that is not a PUT. - */ - lastNotPut: InternalOpId | null; - /** - * Number of REMOVE/MOVE operations seen since lastNotPut. - */ - opsSincePut: number; - /** - * Incrementally-updated checksum, up to maxOpId. - */ - checksum: number; - /** - * Op count for the checksum. - */ - opCount: number; - /** - * Byte size of ops covered by the checksum. - */ - opBytes: number; -} - -type CompactBucketDataDocument = Pick< - TaggedBucketDataDocument, - '_id' | 'def' | 'op' | 'table' | 'row_id' | 'source_table' | 'source_key' | 'checksum' | 'target_op' -> & { - size: number | bigint; -}; - -type CompactClearBucketDataDocument = Pick; -type BucketDataCollectionDocument = BucketDataDocumentV1 | BucketDataDocumentV3; -type BucketDataClearProjection = { - _id: BucketDataDocumentBase['_id']; - op: CompactClearBucketDataDocument['op']; - checksum: bigint; - target_op?: bigint | null; -}; - -export interface MongoCompactOptions extends storage.CompactOptions {} - -const DEFAULT_CLEAR_BATCH_LIMIT = 5000; -const DEFAULT_MOVE_BATCH_LIMIT = 2000; -const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; -const DEFAULT_MIN_BUCKET_CHANGES = 10; -const DEFAULT_MIN_CHANGE_RATIO = 0.1; -const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000; -/** This default is primarily for tests. */ -const DEFAULT_MEMORY_LIMIT_MB = 64; - -export interface DirtyBucket { - bucket: string; - definitionId: BucketDefinitionId | null; - estimatedCount: number; - dirtyRatio?: number; -} - -export abstract class BaseMongoCompactor { - protected updates: mongo.AnyBulkWriteOperation[] = []; - protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; - protected activeBucketDataCollection: mongo.Collection | null = null; - protected activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; - - protected readonly idLimitBytes: number; - protected readonly moveBatchLimit: number; - protected readonly moveBatchQueryLimit: number; - protected readonly clearBatchLimit: number; - protected readonly minBucketChanges: number; - protected readonly minChangeRatio: number; - protected readonly maxOpId: bigint; - protected readonly buckets: string[] | undefined; - protected readonly signal?: AbortSignal; - protected readonly group_id: number; - - constructor( - protected readonly storage: MongoSyncBucketStorage, - protected readonly db: VersionedPowerSyncMongo, - options: MongoCompactOptions - ) { - this.group_id = storage.group_id; - this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; - this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; - this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; - this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; - this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES; - this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO; - this.maxOpId = options.maxOpId ?? 0n; - this.buckets = options.compactBuckets; - this.signal = options.signal; - } - - /** - * Compact buckets by converting operations into MOVE and/or CLEAR operations. - * - * See /docs/compacting-operations.md for details. - */ - async compact() { - if (this.buckets) { - for (const bucket of this.buckets) { - // We can make this more efficient later on by iterating through the buckets in a single query. - // That makes batching more tricky, so we leave for later. - await this.compactSingleBucketRetried(bucket); - } - } else { - await this.compactDirtyBuckets(); - } - } - - /** - * Subset of compact, only populating checksums where relevant. - */ - async populateChecksums(options: { minBucketChanges: number }): Promise { - let count = 0; - while (true) { - this.signal?.throwIfAborted(); - const buckets = await this.dirtyBucketBatchForChecksums(options); - if (buckets.length == 0) { - break; - } - this.signal?.throwIfAborted(); - - const start = Date.now(); - // Filter batch by estimated bucket size, to reduce possibility of timeouts. - const checkBuckets: typeof buckets = []; - let totalCountEstimate = 0; - for (const bucket of buckets) { - checkBuckets.push(bucket); - totalCountEstimate += bucket.estimatedCount; - if (totalCountEstimate > 50_000) { - break; - } - } - logger.info( - `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}` - ); - await this.updateChecksumsBatch(checkBuckets); - logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`); - count += checkBuckets.length; - } - return { buckets: count }; - } - - protected async *dirtyBucketBatchesForCollection( - collection: mongo.Collection, - lastId: TCollectionBucketState['_id'], - maxId: TCollectionBucketState['_id'], - options: { - minBucketChanges: number; - minChangeRatio: number; - }, - getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null - ): AsyncGenerator { - while (true) { - // To avoid timeouts from too many buckets not meeting the minBucketChanges criteria, use an aggregation pipeline - // to scan a fixed batch of buckets at a time, but only return buckets that meet the criteria. - const [result] = await collection - .aggregate<{ - buckets: TCollectionBucketState[]; - cursor: Pick[]; - }>( - [ - { - $match: { - _id: { $gt: lastId, $lt: maxId } - } - }, - { - $sort: { _id: 1 } - }, - { - // Scan a fixed number of docs each query so sparse matches don't block progress. - $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE - }, - { - $facet: { - buckets: [ - { - $match: { - 'estimate_since_compact.count': { $gte: options.minBucketChanges } - } - }, - { - $project: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - } - } - ], - // This is used for the next query. - cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }] - } - } - ], - { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } - ) - .toArray(); - - const cursor = result?.cursor?.[0]; - if (cursor == null) { - break; - } - lastId = cursor._id; - - const mapped = (result?.buckets ?? []).map((bucketState) => { - // The numbers, specifically the bytes, could be a bigint. Convert to Number to allow calculating ratios. - // BigInt precision is not needed here since this is only an estimate. - const updatedCount = bucketState.estimate_since_compact?.count ?? 0; - const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount; - const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0); - const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes; - const dirtyChangeNumber = totalCount > 0 ? updatedCount / totalCount : 0; - const dirtyChangeBytes = totalBytes > 0 ? updatedBytes / totalBytes : 0; - return { - bucket: bucketState._id.b, - definitionId: getDefinitionId(bucketState), - estimatedCount: totalCount, - dirtyRatio: Math.max(dirtyChangeNumber, dirtyChangeBytes) - }; - }); - - yield mapped.filter( - (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio - ); - } - } - - protected async dirtyBucketBatchForChecksumsForCollection( - collection: mongo.Collection, - filter: mongo.Filter, - getDefinitionId: (state: mongo.WithId) => BucketDefinitionId | null - ): Promise { - const dirtyBuckets = await collection - .find(filter, { - projection: { - _id: 1, - estimate_since_compact: 1, - compacted_state: 1 - }, - sort: { - 'estimate_since_compact.count': -1 - }, - limit: 200, - maxTimeMS: MONGO_OPERATION_TIMEOUT_MS - }) - .toArray(); - - return dirtyBuckets.map((bucket) => ({ - bucket: bucket._id.b, - definitionId: getDefinitionId(bucket), - estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0) - })); - } - - public abstract dirtyBucketBatches(options: { - minBucketChanges: number; - minChangeRatio: number; - }): AsyncGenerator; - - public abstract dirtyBucketBatchForChecksums(options: { minBucketChanges: number }): Promise; - - protected async compactDirtyBuckets() { - for await (const buckets of this.dirtyBucketBatches({ - minBucketChanges: this.minBucketChanges, - minChangeRatio: this.minChangeRatio - })) { - this.signal?.throwIfAborted(); - if (buckets.length == 0) { - continue; - } - - for (const { bucket, definitionId } of buckets) { - await this.compactSingleBucketRetried(bucket, definitionId); - } - } - } - - /** - * Compaction for a single bucket, with retries on failure. - * - * This covers against occasional network or other database errors during a long compact job. - */ - protected async compactSingleBucketRetried(bucket: string, definitionId: BucketDefinitionId | null = null) { - let retryCount = 0; - while (true) { - try { - await this.compactSingleBucket(bucket, definitionId); - break; - } catch (e) { - if (retryCount < 3 && isMongoServerError(e)) { - logger.warn(`Error compacting bucket ${bucket}, retrying...`, e); - retryCount++; - await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount)); - } else { - throw e; - } - } - } - } - - protected async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { - const idLimitBytes = this.idLimitBytes; - const bucketCollection = await this.getBucketDataCollection(bucket, definitionId); - if (bucketCollection == null) { - return; - } - this.activeBucketDataCollection = bucketCollection.collection; - this.activeBucketDefinitionId = bucketCollection.definitionId; - try { - const currentState: CurrentBucketState = { - bucket, - definitionId: bucketCollection.definitionId, - seen: new Map(), - trackingSize: 0, - lastNotPut: null, - opsSincePut: 0, - checksum: 0, - opCount: 0, - opBytes: 0 - }; - - // Constant lower bound. - const lowerBound = this.bucketDataKey(bucket, new mongo.MinKey() as any); - // Upper bound is adjusted for each batch. - let upperBound = this.bucketDataKey(bucket, new mongo.MaxKey() as any); - - while (true) { - this.signal?.throwIfAborted(); - - // Query one batch at a time, to avoid cursor timeouts. - const pipeline = [ - { - $match: { - _id: { - $gte: lowerBound, - $lt: upperBound - }, - // Workaround for a clustered collection bug where the $lt operator may include upperBound. - // https://jira.mongodb.org/browse/SERVER-121822 - '_id.o': { $lt: upperBound.o } - } - }, - { $sort: { _id: -1 } }, - { $limit: this.moveBatchQueryLimit }, - { - $project: { - _id: 1, - op: 1, - table: 1, - row_id: 1, - source_table: 1, - source_key: 1, - checksum: 1, - size: { $bsonSize: '$$ROOT' } - } - } - ]; - - const cursor = bucketCollection.collection.aggregate( - pipeline, - { - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: this.moveBatchQueryLimit + 1 - } - ); - // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. - // Instead, we load up to the limit. - const rawBatch = await cursor.toArray(); - const batch = rawBatch.map((document) => this.tagBucketDataDocument(document, bucketCollection.definitionId)); - - if (batch.length == 0) { - // We've reached the end. - break; - } - - // Reuse the exact collection _id value from Mongo for the next bound. - upperBound = rawBatch[rawBatch.length - 1]._id; - - for (const doc of batch) { - if (doc._id.o > this.maxOpId) { - continue; - } - - currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum)); - currentState.opCount += 1; - - let isPersistentPut = doc.op == 'PUT'; - - currentState.opBytes += Number(doc.size); - if (doc.op == 'REMOVE' || doc.op == 'PUT') { - const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; - const targetOp = currentState.seen.get(key); - if (targetOp) { - // Will convert to MOVE, so don't count as PUT. - isPersistentPut = false; - - this.updates.push({ - updateOne: { - filter: { _id: this.bucketDataKey(doc._id.b, doc._id.o) }, - update: { - $set: { - op: 'MOVE', - target_op: targetOp - }, - $unset: { - source_table: 1, - source_key: 1, - table: 1, - row_id: 1, - data: 1 - } - } satisfies mongo.UpdateFilter - } - }); - - // TODO: better estimate for this. - currentState.opBytes += 200 - Number(doc.size); - } else if (currentState.trackingSize < idLimitBytes) { - // flatstr reduces the memory usage by flattening the string. - currentState.seen.set(utils.flatstr(key), doc._id.o); - // length + 16 for the string - // 24 for the bigint - // 50 for map overhead - // 50 for additional overhead - currentState.trackingSize += key.length + 140; - } - } - - if (isPersistentPut) { - currentState.lastNotPut = null; - currentState.opsSincePut = 0; - } else if (doc.op != 'CLEAR') { - if (currentState.lastNotPut == null) { - currentState.lastNotPut = doc._id.o; - } - currentState.opsSincePut += 1; - } - - if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { - await this.flush(); - } - } - - logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); - } - - // Free memory before clearing the bucket. - currentState.seen.clear(); - if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { - logger.info( - `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` - ); - // Need flush() before clear(). - await this.flush(); - await this.clearBucket(currentState); - } - - // Do this after clearBucket so we have accurate counts. - this.updateBucketChecksums(currentState); - // Need another flush after updateBucketChecksums(). - await this.flush(); - } finally { - this.activeBucketDataCollection = null; - this.activeBucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; - } - } - - protected updateBucketChecksums(state: CurrentBucketState) { - if (state.opCount < 0) { - throw new ServiceAssertionError( - `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}` - ); - } - this.bucketStateUpdates.push({ - updateOne: { - filter: this.bucketStateFilter(state.bucket, state.definitionId), - update: { - $set: { - compacted_state: { - op_id: this.maxOpId, - count: state.opCount, - checksum: BigInt(state.checksum), - bytes: state.opBytes - }, - estimate_since_compact: { - // There could have been a whole bunch of new operations added to the bucket while compacting, - // which we don't currently cater for. We could potentially query for that, but that adds overhead. - count: 0, - bytes: 0 - } - } - } satisfies mongo.UpdateFilter, - // We generally expect this to have been created before. - // We don't create new ones here, to avoid issues with the unique index on bucket_updates. - upsert: false - } - }); - } - - protected async flush() { - if (this.updates.length > 0) { - logger.info(`Compacting ${this.updates.length} ops`); - if (this.activeBucketDataCollection == null) { - throw new ServiceAssertionError('No bucket_data collection selected for compaction'); - } - await this.activeBucketDataCollection.bulkWrite(this.updates, { - // Order is not important. Since checksums are not affected, these operations can happen in any order, - // and it's fine if the operations are partially applied. Each individual operation is atomic. - ordered: false - }); - this.updates = []; - } - if (this.bucketStateUpdates.length > 0) { - logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); - await this.flushBucketStateUpdates(); - this.bucketStateUpdates = []; - } - } - - /** - * Perform a CLEAR compact for a bucket. - * - * @param currentState tracks the last non-PUT op, which will be converted to CLEAR. - */ - protected async clearBucket(currentState: CurrentBucketState) { - const bucket = currentState.bucket; - const clearOp = currentState.lastNotPut!; - const bucketCollection = this.activeBucketDataCollection; - if (bucketCollection == null) { - throw new ServiceAssertionError('No bucket_data collection selected for compaction'); - } - - const opFilter = { - _id: { - $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), - $lte: this.bucketDataKey(bucket, clearOp) - } - }; - - const session = this.db.client.startSession(); - try { - let done = false; - while (!done) { - this.signal?.throwIfAborted(); - let opCountDiff = 0; - // Do the CLEAR operation in batches, with each batch a separate transaction. - // The state after each batch is fully consistent. - // We need a transaction per batch to make sure checksums stay consistent. - await session.withTransaction( - async () => { - const query = bucketCollection.find(opFilter as any, { - session, - sort: { _id: 1 }, - projection: { - _id: 1, - op: 1, - checksum: 1, - target_op: 1 - }, - limit: this.clearBatchLimit - }); - let checksum = 0; - let lastOp: CompactClearBucketDataDocument | null = null; - let targetOp: bigint | null = null; - let gotAnOp = false; - let numberOfOpsToClear = 0; - for await (const rawOp of query.stream()) { - const op = this.tagClearBucketDataDocument( - rawOp as BucketDataClearProjection, - this.activeBucketDefinitionId - ); - - if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') { - checksum = utils.addChecksums(checksum, Number(op.checksum)); - lastOp = op; - numberOfOpsToClear += 1; - if (op.op != 'CLEAR') { - gotAnOp = true; - } - if (op.target_op != null && (targetOp == null || op.target_op > targetOp)) { - targetOp = op.target_op; - } - } else { - throw new ReplicationAssertionError( - `Unexpected ${op.op} operation at ${this.formatBucketDataKey(op._id)}` - ); - } - } - if (!gotAnOp) { - done = true; - return; - } - - logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?._id.o}`); - await bucketCollection.deleteMany( - { - _id: { - $gte: this.bucketDataKey(bucket, new mongo.MinKey()), - $lte: this.bucketDataKey(lastOp!._id.b, lastOp!._id.o) - } - }, - { session } - ); - - await bucketCollection.insertOne( - this.collectionBucketDataDocument({ - def: this.activeBucketDefinitionId, - _id: lastOp!._id, - op: 'CLEAR', - checksum: BigInt(checksum), - data: null, - target_op: targetOp - }), - { session } - ); - - opCountDiff = -numberOfOpsToClear + 1; - }, - { - writeConcern: { w: 'majority' }, - readConcern: { level: 'snapshot' } - } - ); - // Update outside the transaction, since the transaction can be retried multiple times. - currentState.opCount += opCountDiff; - } - } finally { - await session.endSession(); - } - } - - protected async updateChecksumsBatch(buckets: Pick[]) { - const checksums = await this.computeChecksumsForBuckets(buckets); - const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId])); - - for (const bucketChecksum of checksums.values()) { - if (isPartialChecksum(bucketChecksum)) { - // Should never happen since we don't specify `start`. - throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`); - } - - this.bucketStateUpdates.push({ - updateOne: { - filter: this.bucketStateFilter( - bucketChecksum.bucket, - definitionIdByBucket.get(bucketChecksum.bucket) ?? null - ), - update: { - $set: { - compacted_state: { - op_id: this.maxOpId, - count: bucketChecksum.count, - checksum: BigInt(bucketChecksum.checksum), - bytes: null - }, - estimate_since_compact: { - count: 0, - bytes: 0 - } - } - } satisfies mongo.UpdateFilter, - // We don't create new ones here - it gets tricky to get the last_op right with the unique index on - // bucket_updates. - upsert: false - } - }); - } - - await this.flush(); - } - - protected tagBucketDataDocument( - document: BucketDataCollectionDocument & { size: number | bigint }, - definitionId: BucketDefinitionId - ): CompactBucketDataDocument { - const tagged = bucketDataDocumentToTagged(document, definitionId); - return { - ...tagged, - size: document.size - }; - } - - protected tagClearBucketDataDocument( - document: BucketDataClearProjection, - definitionId: BucketDefinitionId - ): CompactClearBucketDataDocument { - return { - def: definitionId, - _id: { - b: document._id.b, - o: document._id.o - }, - op: document.op, - checksum: document.checksum, - target_op: document.target_op - }; - } - - protected formatBucketDataKey(key: BucketDataDocumentBase['_id'] | { _id: BucketDataDocumentBase['_id'] }) { - const bucket = 'b' in key ? key.b : key._id.b; - const op = 'o' in key ? key.o : key._id.o; - return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; - } - - protected abstract flushBucketStateUpdates(): Promise; - protected abstract computeChecksumsForBuckets( - buckets: Pick[] - ): Promise; - protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; - protected abstract bucketDataKey( - bucket: string, - opId: InternalOpId | mongo.MinKey | mongo.MaxKey - ): BucketDataDocumentBase['_id']; - protected abstract getBucketDataCollection( - bucket: string, - definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null>; - protected abstract collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase; -} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts index 83bd721e2..072dc9c5f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts @@ -36,7 +36,7 @@ import { StorageConfig } from '../models.js'; import { MongoChecksumOptions, MongoChecksums } from '../MongoChecksums.js'; -import { MongoCompactor } from '../MongoCompactor.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoWriteCheckpointAPI } from '../MongoWriteCheckpointAPI.js'; @@ -94,6 +94,7 @@ export abstract class BaseMongoSyncBucketStorage } protected abstract createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums; + protected abstract createMongoCompactor(options: MongoCompactOptions): MongoCompactor; get writeCheckpointMode() { return this.writeCheckpointAPI.writeCheckpointMode; @@ -512,7 +513,7 @@ export abstract class BaseMongoSyncBucketStorage const checkpoint = await this.getCheckpointInternal(); maxOpId = checkpoint?.checkpoint ?? undefined; } - await new MongoCompactor(this as any, this.db, { ...options, maxOpId }).compact(); + await this.createMongoCompactor({ ...options, maxOpId }).compact(); if (maxOpId != null && options?.compactParameterData) { await new MongoParameterCompactor(this.db, this.group_id, maxOpId, options).compact(); @@ -522,7 +523,7 @@ export abstract class BaseMongoSyncBucketStorage async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise { logger.info(`Populating persistent checksum cache...`); const start = Date.now(); - const compactor = new MongoCompactor(this as any, this.db, { + const compactor = this.createMongoCompactor({ ...options, memoryLimitMB: 0 }); diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index cf5131d6d..0fb31725b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -12,11 +12,11 @@ import { taggedBucketDataDocumentToV1 } from '../models.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { BaseMongoCompactor, DirtyBucket } from '../common/MongoCompactorBase.js'; +import { DirtyBucket, MongoCompactor } from '../common/MongoCompactor.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; import { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; -export class MongoCompactorV1 extends BaseMongoCompactor { +export class MongoCompactorV1 extends MongoCompactor { // Override types to the more specific ones declare protected readonly db: VersionedPowerSyncMongoV1; declare protected readonly storage: MongoSyncBucketStorageV1; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 85459216b..928429c0c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -16,6 +16,7 @@ import * as bson from 'bson'; import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorageBase.js'; @@ -33,6 +34,7 @@ import { } from '../models.js'; import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; import { MongoChecksumsV1 } from './MongoChecksumsV1.js'; +import { MongoCompactorV1 } from './MongoCompactorV1.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { @@ -65,6 +67,10 @@ export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { }); } + protected createMongoCompactor(options: MongoCompactOptions): MongoCompactor { + return new MongoCompactorV1(this, this.db, options); + } + protected sourceTableBaseId(): Partial { return { group_id: this.group_id }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 6dad4644f..ad61a38b3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -2,7 +2,7 @@ import { MONGO_OPERATION_TIMEOUT_MS, mongo } from '@powersync/lib-service-mongod import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { BaseMongoCompactor, DirtyBucket } from '../common/MongoCompactorBase.js'; +import { DirtyBucket, MongoCompactor } from '../common/MongoCompactor.js'; import { BucketDataDocumentBase, BucketDataKeyV3, @@ -14,7 +14,7 @@ import { import { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; -export class MongoCompactorV3 extends BaseMongoCompactor { +export class MongoCompactorV3 extends MongoCompactor { declare protected readonly db: VersionedPowerSyncMongoV3; declare protected readonly storage: MongoSyncBucketStorageV3; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index b591fb5d9..86e9bf5fc 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -28,6 +28,8 @@ import { CommonSourceTableDocument } from '../models.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoCompactorV3 } from './MongoCompactorV3.js'; export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { // Declare types to be more specific @@ -80,6 +82,10 @@ export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { }); } + protected createMongoCompactor(options: MongoCompactOptions): MongoCompactor { + return new MongoCompactorV3(this, this.db, options); + } + protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch { return new MongoBucketBatchV3(batchOptions); } diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index fed79a826..959e85c52 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1,7 +1,8 @@ import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; -import { MongoCompactor } from '../../src/storage/implementation/MongoCompactor.js'; +import { MongoCompactorV1 } from '../../src/storage/implementation/v1/MongoCompactorV1.js'; +import { MongoCompactorV3 } from '../../src/storage/implementation/v3/MongoCompactorV3.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; describe('Mongo Sync Bucket Storage Compact', () => { @@ -214,9 +215,12 @@ bucket_definitions: // This test uses a couple of internal APIs of the compactor - there is no simple way // to test this using the current public APIs. - const compactor = new MongoCompactor(bucketStorage, (bucketStorage as any).db, { - maxOpId: 5n - }); + let compactor: MongoCompactorV1 | MongoCompactorV3; + if (storageDb.storageConfig.incrementalReprocessing) { + compactor = new MongoCompactorV3(bucketStorage as any, storageDb, { maxOpId: 5n }); + } else { + compactor = new MongoCompactorV1(bucketStorage as any, storageDb, { maxOpId: 5n }); + } const dirtyBuckets = (compactor as any).dirtyBucketBatches({ minBucketChanges: 1, From 7a8d84e1bf494caac8dd3c4941d0c3c86238f65c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 12:51:22 +0200 Subject: [PATCH 67/93] Remove compatibility re-rexports. --- .../src/storage/MongoBucketStorage.ts | 2 +- .../src/storage/implementation/MongoChecksums.ts | 1 - .../src/storage/implementation/MongoCompactor.ts | 1 - .../src/storage/implementation/MongoParameterCompactor.ts | 1 - .../src/storage/implementation/OperationBatch.ts | 2 +- .../implementation/common/MongoSyncBucketStorageBase.ts | 6 +++--- .../src/storage/implementation/v1/MongoChecksumsV1.ts | 2 +- .../src/storage/implementation/v1/MongoCompactorV1.ts | 2 +- .../storage/implementation/v1/MongoSyncBucketStorageV1.ts | 4 ++-- .../src/storage/implementation/v3/MongoChecksumsV3.ts | 2 +- .../src/storage/implementation/v3/MongoCompactorV3.ts | 2 +- .../storage/implementation/v3/MongoSyncBucketStorageV3.ts | 4 ++-- 12 files changed, 13 insertions(+), 16 deletions(-) delete mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts delete mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts delete mode 100644 modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 8b8667b4e..a2b2a1b44 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -13,7 +13,7 @@ import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedS import { createMongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; import type { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; import { generateSlotName } from '../utils/util.js'; -import { MongoChecksumOptions } from './implementation/MongoChecksums.js'; +import { MongoChecksumOptions } from './implementation/common/MongoChecksums.js'; import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; export interface MongoBucketStorageOptions { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts deleted file mode 100644 index 05370c74a..000000000 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './common/MongoChecksums.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts deleted file mode 100644 index 46ea63083..000000000 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './common/MongoCompactor.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts deleted file mode 100644 index 0537f0397..000000000 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './common/MongoParameterCompactor.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/OperationBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/OperationBatch.ts index 95193042f..33d5209d2 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/OperationBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/OperationBatch.ts @@ -2,7 +2,7 @@ import { ToastableSqliteRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { storage } from '@powersync/service-core'; -import { mongoTableId } from '../storage-index.js'; +import { mongoTableId } from '../../utils/util.js'; /** * Maximum number of operations in a batch. diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts index 072dc9c5f..a0a43f361 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts @@ -35,9 +35,9 @@ import { SourceKey, StorageConfig } from '../models.js'; -import { MongoChecksumOptions, MongoChecksums } from '../MongoChecksums.js'; -import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; -import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; +import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js'; +import { MongoParameterCompactor } from './MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoWriteCheckpointAPI } from '../MongoWriteCheckpointAPI.js'; import { MongoSyncBucketStorageContext } from './MongoSyncBucketStorageContext.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts index c39955ffe..894af7115 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts @@ -6,7 +6,7 @@ import { PartialChecksumMap } from '@powersync/service-core'; import { FetchPartialBucketChecksumByBucket } from '../common/MongoChecksums.js'; -import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoChecksums } from '../common/MongoChecksums.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoChecksumsV1 extends MongoChecksums { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 0fb31725b..986659919 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -14,7 +14,7 @@ import { import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { DirtyBucket, MongoCompactor } from '../common/MongoCompactor.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; -import { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; +import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; export class MongoCompactorV1 extends MongoCompactor { // Override types to the more specific ones diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 928429c0c..fddcf29f7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -15,8 +15,8 @@ import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-ru import * as bson from 'bson'; import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; -import { MongoChecksums } from '../MongoChecksums.js'; -import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoChecksums } from '../common/MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from '../common/MongoCompactor.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorageBase.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index 6597be833..5df1c8494 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -12,7 +12,7 @@ import { FetchPartialBucketChecksumV3, MongoChecksumOptions } from '../common/MongoChecksums.js'; -import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoChecksums } from '../common/MongoChecksums.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoChecksumsV3 extends MongoChecksums { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index ad61a38b3..a3855a7c7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -11,7 +11,7 @@ import { TaggedBucketDataDocument, taggedBucketDataDocumentToV3 } from '../models.js'; -import { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; +import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoCompactorV3 extends MongoCompactor { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 86e9bf5fc..7b5864a37 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -27,8 +27,8 @@ import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; import { CommonSourceTableDocument } from '../models.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; -import { MongoChecksums } from '../MongoChecksums.js'; -import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoChecksums } from '../common/MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from '../common/MongoCompactor.js'; import { MongoCompactorV3 } from './MongoCompactorV3.js'; export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { From 214dda427395218e87bc825633f7a0b41e6947ce Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 12:55:37 +0200 Subject: [PATCH 68/93] Remove MongoParameterCompactorBase. --- .../common/MongoParameterCompactor.ts | 140 ++++++++++++++++-- .../common/MongoParameterCompactorBase.ts | 131 ---------------- .../common/MongoSyncBucketStorageBase.ts | 6 +- .../v1/MongoParameterCompactorV1.ts | 4 +- .../v1/MongoSyncBucketStorageV1.ts | 9 ++ .../v3/MongoParameterCompactorV3.ts | 4 +- .../v3/MongoSyncBucketStorageV3.ts | 9 ++ 7 files changed, 152 insertions(+), 151 deletions(-) delete mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts index 06d17ecd6..e2a8be7ef 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts @@ -1,21 +1,131 @@ -import { CompactOptions, InternalOpId } from '@powersync/service-core'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { logger } from '@powersync/lib-services-framework'; +import { bson, CompactOptions, InternalOpId } from '@powersync/service-core'; +import { LRUCache } from 'lru-cache'; import type { VersionedPowerSyncMongo } from '../db.js'; -import { MongoParameterCompactorV1 } from '../v1/MongoParameterCompactorV1.js'; -import { MongoParameterCompactorV3 } from '../v3/MongoParameterCompactorV3.js'; -import { BaseMongoParameterCompactor } from './MongoParameterCompactorBase.js'; - -export class MongoParameterCompactor { - private readonly impl: BaseMongoParameterCompactor; - - constructor(db: VersionedPowerSyncMongo, group_id: number, checkpoint: InternalOpId, options: CompactOptions) { - if (db.storageConfig.incrementalReprocessing) { - this.impl = new MongoParameterCompactorV3(db, group_id, checkpoint, options); - } else { - this.impl = new MongoParameterCompactorV1(db, group_id, checkpoint, options); + +type ParameterCompactionReadDocument = { + _id: InternalOpId; + key: mongo.Document; + lookup: unknown; + bucket_parameters?: unknown[] | null; +}; + +/** + * Compacts parameter lookup data (the bucket_parameters collection). + * + * This scans through the entire collection to find data to compact. + * + * For background, see the `/docs/parameters-lookups.md` file. + */ +export abstract class MongoParameterCompactor { + constructor( + protected readonly db: VersionedPowerSyncMongo, + protected readonly group_id: number, + protected readonly checkpoint: InternalOpId, + protected readonly options: CompactOptions + ) {} + + async compact() { + logger.info(`Compacting parameters for sync config ${this.group_id} up to checkpoint ${this.checkpoint}`); + for (const collection of await this.getCollections()) { + await this.compactCollection(collection); } } - async compact() { - return this.impl.compact(); + protected abstract getCollections(): Promise[]>; + + protected abstract collectionFilter(): mongo.Document; + + protected abstract deleteFilter(doc: mongo.Document): mongo.Document; + + protected async compactCollection(collection: mongo.Collection) { + // This is the currently-active checkpoint. + // We do not remove any data that may be used by this checkpoint. + // snapshot queries ensure that if any clients are still using older checkpoints, they would + // not be affected by this compaction. + const checkpoint = this.checkpoint; + + // Index on {'key.g': 1, lookup: 1, _id: 1} + // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup) + // in MongoDB already. However, that risks running into cases where MongoDB needs to process + // very large amounts of data before returning results, which could lead to timeouts. + const cursor = collection.find(this.collectionFilter(), { + sort: { lookup: 1, _id: 1 }, + batchSize: 10_000, + projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 } + }); + + // The index doesn't cover sorting by key, so we keep our own cache of the last seen key. + let lastByKey = new LRUCache({ + max: this.options.compactParameterCacheLimit ?? 10_000 + }); + let removeIds: InternalOpId[] = []; + let removeDeleted: mongo.AnyBulkWriteOperation[] = []; + let checkedEntries = 0; + let checkedEntriesAtLastLog = 0; + let lastProgressLogTime = Date.now(); + + const flush = async (force: boolean) => { + if (removeIds.length >= 1000 || (force && removeIds.length > 0)) { + const results = await collection.deleteMany({ _id: { $in: removeIds } } as any); + logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`); + removeIds = []; + } + + if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) { + const results = await collection.bulkWrite(removeDeleted); + logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`); + removeDeleted = []; + } + }; + + while (await cursor.hasNext()) { + const batch = cursor.readBufferedDocuments() as unknown as ParameterCompactionReadDocument[]; + checkedEntries += batch.length; + const now = Date.now(); + if (now - lastProgressLogTime >= 60_000) { + const elapsedSeconds = (now - lastProgressLogTime) / 1000; + const rate = (checkedEntries - checkedEntriesAtLastLog) / elapsedSeconds; + logger.info(`Checked ${checkedEntries} parameter index entries for compaction (${rate.toFixed(1)} entries/s)`); + lastProgressLogTime = now; + checkedEntriesAtLastLog = checkedEntries; + } + + for (const doc of batch) { + if (doc._id >= checkpoint) { + continue; + } + const uniqueKey = ( + bson.serialize({ + k: doc.key, + l: doc.lookup + }) as Buffer + ).toString('base64'); + const previous = lastByKey.get(uniqueKey); + if (previous != null && previous < doc._id) { + // We have a newer entry for the same key, so we can remove the old one. + removeIds.push(previous); + } + lastByKey.set(uniqueKey, doc._id); + + if (doc.bucket_parameters?.length == 0) { + // This is a delete operation, so we can remove it completely. + // For this we cannot remove the operation itself only: There is a possibility that + // there is still an earlier operation with the same key and lookup, that we don't have + // in the cache due to cache size limits. So we need to explicitly remove all earlier operations. + removeDeleted.push({ + deleteMany: { + filter: this.deleteFilter(doc) + } + }); + } + } + + await flush(false); + } + + await flush(true); + logger.info(`Parameter compaction completed for ${collection.collectionName}`); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts deleted file mode 100644 index b1c3f29aa..000000000 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactorBase.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { mongo } from '@powersync/lib-service-mongodb'; -import { logger } from '@powersync/lib-services-framework'; -import { bson, CompactOptions, InternalOpId } from '@powersync/service-core'; -import { LRUCache } from 'lru-cache'; -import type { VersionedPowerSyncMongo } from '../db.js'; - -type ParameterCompactionReadDocument = { - _id: InternalOpId; - key: mongo.Document; - lookup: unknown; - bucket_parameters?: unknown[] | null; -}; - -/** - * Compacts parameter lookup data (the bucket_parameters collection). - * - * This scans through the entire collection to find data to compact. - * - * For background, see the `/docs/parameters-lookups.md` file. - */ -export abstract class BaseMongoParameterCompactor { - constructor( - protected readonly db: VersionedPowerSyncMongo, - protected readonly group_id: number, - protected readonly checkpoint: InternalOpId, - protected readonly options: CompactOptions - ) {} - - async compact() { - logger.info(`Compacting parameters for sync config ${this.group_id} up to checkpoint ${this.checkpoint}`); - for (const collection of await this.getCollections()) { - await this.compactCollection(collection); - } - } - - protected abstract getCollections(): Promise[]>; - - protected abstract collectionFilter(): mongo.Document; - - protected abstract deleteFilter(doc: mongo.Document): mongo.Document; - - protected async compactCollection(collection: mongo.Collection) { - // This is the currently-active checkpoint. - // We do not remove any data that may be used by this checkpoint. - // snapshot queries ensure that if any clients are still using older checkpoints, they would - // not be affected by this compaction. - const checkpoint = this.checkpoint; - - // Index on {'key.g': 1, lookup: 1, _id: 1} - // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup) - // in MongoDB already. However, that risks running into cases where MongoDB needs to process - // very large amounts of data before returning results, which could lead to timeouts. - const cursor = collection.find(this.collectionFilter(), { - sort: { lookup: 1, _id: 1 }, - batchSize: 10_000, - projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 } - }); - - // The index doesn't cover sorting by key, so we keep our own cache of the last seen key. - let lastByKey = new LRUCache({ - max: this.options.compactParameterCacheLimit ?? 10_000 - }); - let removeIds: InternalOpId[] = []; - let removeDeleted: mongo.AnyBulkWriteOperation[] = []; - let checkedEntries = 0; - let checkedEntriesAtLastLog = 0; - let lastProgressLogTime = Date.now(); - - const flush = async (force: boolean) => { - if (removeIds.length >= 1000 || (force && removeIds.length > 0)) { - const results = await collection.deleteMany({ _id: { $in: removeIds } } as any); - logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`); - removeIds = []; - } - - if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) { - const results = await collection.bulkWrite(removeDeleted); - logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`); - removeDeleted = []; - } - }; - - while (await cursor.hasNext()) { - const batch = cursor.readBufferedDocuments() as unknown as ParameterCompactionReadDocument[]; - checkedEntries += batch.length; - const now = Date.now(); - if (now - lastProgressLogTime >= 60_000) { - const elapsedSeconds = (now - lastProgressLogTime) / 1000; - const rate = (checkedEntries - checkedEntriesAtLastLog) / elapsedSeconds; - logger.info(`Checked ${checkedEntries} parameter index entries for compaction (${rate.toFixed(1)} entries/s)`); - lastProgressLogTime = now; - checkedEntriesAtLastLog = checkedEntries; - } - - for (const doc of batch) { - if (doc._id >= checkpoint) { - continue; - } - const uniqueKey = ( - bson.serialize({ - k: doc.key, - l: doc.lookup - }) as Buffer - ).toString('base64'); - const previous = lastByKey.get(uniqueKey); - if (previous != null && previous < doc._id) { - // We have a newer entry for the same key, so we can remove the old one. - removeIds.push(previous); - } - lastByKey.set(uniqueKey, doc._id); - - if (doc.bucket_parameters?.length == 0) { - // This is a delete operation, so we can remove it completely. - // For this we cannot remove the operation itself only: There is a possibility that - // there is still an earlier operation with the same key and lookup, that we don't have - // in the cache due to cache size limits. So we need to explicitly remove all earlier operations. - removeDeleted.push({ - deleteMany: { - filter: this.deleteFilter(doc) - } - }); - } - } - - await flush(false); - } - - await flush(true); - logger.info(`Parameter compaction completed for ${collection.collectionName}`); - } -} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts index a0a43f361..b185d2c69 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts @@ -95,6 +95,10 @@ export abstract class BaseMongoSyncBucketStorage protected abstract createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums; protected abstract createMongoCompactor(options: MongoCompactOptions): MongoCompactor; + protected abstract createMongoParameterCompactor( + checkpoint: InternalOpId, + options: storage.CompactOptions + ): MongoParameterCompactor; get writeCheckpointMode() { return this.writeCheckpointAPI.writeCheckpointMode; @@ -516,7 +520,7 @@ export abstract class BaseMongoSyncBucketStorage await this.createMongoCompactor({ ...options, maxOpId }).compact(); if (maxOpId != null && options?.compactParameterData) { - await new MongoParameterCompactor(this.db, this.group_id, maxOpId, options).compact(); + await this.createMongoParameterCompactor(maxOpId, options).compact(); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts index be6cb90e9..f7053ea64 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts @@ -1,8 +1,8 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { BaseMongoParameterCompactor } from '../common/MongoParameterCompactorBase.js'; +import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; -export class MongoParameterCompactorV1 extends BaseMongoParameterCompactor { +export class MongoParameterCompactorV1 extends MongoParameterCompactor { declare protected readonly db: VersionedPowerSyncMongoV1; protected async getCollections(): Promise[]> { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index fddcf29f7..19c66c527 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -17,6 +17,7 @@ import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } f import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { MongoChecksums } from '../common/MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from '../common/MongoCompactor.js'; +import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorageBase.js'; @@ -35,6 +36,7 @@ import { import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; import { MongoChecksumsV1 } from './MongoChecksumsV1.js'; import { MongoCompactorV1 } from './MongoCompactorV1.js'; +import { MongoParameterCompactorV1 } from './MongoParameterCompactorV1.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { @@ -71,6 +73,13 @@ export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { return new MongoCompactorV1(this, this.db, options); } + protected createMongoParameterCompactor( + checkpoint: InternalOpId, + options: storage.CompactOptions + ): MongoParameterCompactor { + return new MongoParameterCompactorV1(this.db, this.group_id, checkpoint, options); + } + protected sourceTableBaseId(): Partial { return { group_id: this.group_id }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts index bce34750a..ecb935a35 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts @@ -1,8 +1,8 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { BaseMongoParameterCompactor } from '../common/MongoParameterCompactorBase.js'; +import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; -export class MongoParameterCompactorV3 extends BaseMongoParameterCompactor { +export class MongoParameterCompactorV3 extends MongoParameterCompactor { declare protected readonly db: VersionedPowerSyncMongoV3; protected async getCollections(): Promise[]> { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 7b5864a37..b76c2649b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -29,7 +29,9 @@ import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; import { MongoChecksums } from '../common/MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from '../common/MongoCompactor.js'; +import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; import { MongoCompactorV3 } from './MongoCompactorV3.js'; +import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js'; export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { // Declare types to be more specific @@ -86,6 +88,13 @@ export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { return new MongoCompactorV3(this, this.db, options); } + protected createMongoParameterCompactor( + checkpoint: InternalOpId, + options: storage.CompactOptions + ): MongoParameterCompactor { + return new MongoParameterCompactorV3(this.db, this.group_id, checkpoint, options); + } + protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch { return new MongoBucketBatchV3(batchOptions); } From dce5dbad742425dc62c71fe659ba8511788eb63c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 12:58:55 +0200 Subject: [PATCH 69/93] Rename MongoSyncBucketStorage. --- .../src/storage/MongoBucketStorage.ts | 4 ++-- .../implementation/common/MongoCompactor.ts | 2 +- ...StorageBase.ts => MongoSyncBucketStorage.ts} | 4 ++-- ...orage.ts => createMongoSyncBucketStorage.ts} | 17 ++++------------- .../v1/MongoSyncBucketStorageV1.ts | 4 ++-- .../v3/MongoSyncBucketStorageV3.ts | 4 ++-- .../src/storage/storage-index.ts | 2 +- .../test/src/storage_sync.test.ts | 2 +- 8 files changed, 15 insertions(+), 24 deletions(-) rename modules/module-mongodb-storage/src/storage/implementation/common/{MongoSyncBucketStorageBase.ts => MongoSyncBucketStorage.ts} (99%) rename modules/module-mongodb-storage/src/storage/implementation/{MongoSyncBucketStorage.ts => createMongoSyncBucketStorage.ts} (64%) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index a2b2a1b44..5669e2589 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -10,8 +10,8 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { PowerSyncMongo } from './implementation/db.js'; import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js'; import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js'; -import { createMongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; -import type { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js'; +import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; +import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; import { generateSlotName } from '../utils/util.js'; import { MongoChecksumOptions } from './implementation/common/MongoChecksums.js'; import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts index 710b58a9e..54cbc7c74 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts @@ -21,7 +21,7 @@ import { bucketDataDocumentToTagged } from '../models.js'; import { cacheKey } from '../OperationBatch.js'; -import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js'; +import { MongoSyncBucketStorage } from '../createMongoSyncBucketStorage.js'; interface CurrentBucketState { /** Bucket name */ diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts similarity index 99% rename from modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts rename to modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts index b185d2c69..2b12e0f2a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts @@ -64,7 +64,7 @@ interface InternalCheckpointChanges extends CheckpointChanges { */ const CHECKPOINT_TIMEOUT_MS = 60_000; -export abstract class BaseMongoSyncBucketStorage +export abstract class MongoSyncBucketStorage extends BaseObserver implements storage.SyncRulesBucketStorage { @@ -764,7 +764,7 @@ export abstract class BaseMongoSyncBucketStorage class MongoReplicationCheckpoint implements ReplicationCheckpoint { constructor( - private storage: BaseMongoSyncBucketStorage, + private storage: MongoSyncBucketStorage, public readonly checkpoint: InternalOpId, public readonly lsn: string | null, public snapshotTime: mongo.Timestamp diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts similarity index 64% rename from modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts rename to modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts index 2062c5ed6..712f2f7d4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts @@ -1,22 +1,13 @@ -import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; -import { - GetCheckpointChangesOptions, - PopulateChecksumCacheOptions, - PopulateChecksumCacheResults, - storage, - utils, - WatchWriteCheckpointOptions -} from '@powersync/service-core'; -import * as bson from 'bson'; +import { storage } from '@powersync/service-core'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; -import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorageBase.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorage.js'; import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js'; import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js'; -export { MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorageBase.js'; +export { MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorage.js'; -export type MongoSyncBucketStorage = BaseMongoSyncBucketStorage; +export type { MongoSyncBucketStorage }; export function createMongoSyncBucketStorage( factory: MongoBucketStorage, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 19c66c527..d559e88d4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -20,7 +20,7 @@ import { MongoCompactOptions, MongoCompactor } from '../common/MongoCompactor.js import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; -import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorageBase.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorage.js'; import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext @@ -39,7 +39,7 @@ import { MongoCompactorV1 } from './MongoCompactorV1.js'; import { MongoParameterCompactorV1 } from './MongoParameterCompactorV1.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; -export class MongoSyncBucketStorageV1 extends BaseMongoSyncBucketStorage { +export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { // Declare types to be more specific declare readonly db: VersionedPowerSyncMongoV1; declare readonly checksums: MongoChecksumsV1; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index b76c2649b..e71fc00f9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -19,7 +19,7 @@ import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js'; -import { BaseMongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorageBase.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorage.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; @@ -33,7 +33,7 @@ import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; import { MongoCompactorV3 } from './MongoCompactorV3.js'; import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js'; -export class MongoSyncBucketStorageV3 extends BaseMongoSyncBucketStorage { +export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { // Declare types to be more specific declare readonly db: VersionedPowerSyncMongoV3; declare readonly checksums: MongoChecksumsV3; diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index 3f554fe98..7d29df5b0 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -5,7 +5,7 @@ export * from './implementation/MongoIdSequence.js'; export * from './implementation/MongoPersistedSyncRules.js'; export * from './implementation/MongoPersistedSyncRulesContent.js'; export * from './implementation/MongoStorageProvider.js'; -export * from './implementation/MongoSyncBucketStorage.js'; +export * from './implementation/createMongoSyncBucketStorage.js'; export * from './implementation/MongoSyncRulesLock.js'; export * from './implementation/OperationBatch.js'; export * from './implementation/common/PersistedBatch.js'; diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index b8dbbc141..0b6e244da 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -5,7 +5,7 @@ import { RequestParameters } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; -import { MongoSyncBucketStorage } from '../../src/storage/implementation/MongoSyncBucketStorage.js'; +import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js'; import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; import { CurrentBucketV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; From 197a9b996f80e53e1ab63edaade7e00164945e27 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 13:04:49 +0200 Subject: [PATCH 70/93] Simplify db. --- .../src/storage/implementation/db.ts | 34 +------------------ .../implementation/v3/MongoChecksumsV3.ts | 2 +- .../implementation/v3/MongoCompactorV3.ts | 2 +- .../v3/MongoSyncBucketStorageV3.ts | 4 +-- .../implementation/v3/PersistedBatchV3.ts | 2 +- .../v3/VersionedPowerSyncMongoV3.ts | 19 ++++++----- .../test/src/storage_sync.test.ts | 2 +- 7 files changed, 18 insertions(+), 47 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index b4b0c2b56..104ac9b4f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -3,26 +3,20 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { POWERSYNC_VERSION, storage } from '@powersync/service-core'; import { MongoStorageConfig } from '../../types/types.js'; -import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; import { BaseVersionedPowerSyncMongo } from './common/VersionedPowerSyncMongoBase.js'; import { BucketDataDocumentV1, BucketDataDocumentV3, BucketParameterDocument, - BucketParameterDocumentV3, - BucketStateDocument, BucketStateDocumentV1, - BucketStateDocumentV3, CheckpointEventDocument, ClientConnectionDocument, CommonSourceTableDocument, CurrentDataDocument, - CurrentDataDocumentV3, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, SourceTableDocument, - SourceTableDocumentV3, StorageConfig, SyncRuleDocument, WriteCheckpointDocument @@ -86,14 +80,6 @@ export class PowerSyncMongo { return new VersionedPowerSyncMongoV1(this, storageConfig); } - bucketDataCollectionNameV3(groupId: number, definitionId: BucketDefinitionId) { - return `bucket_data_${groupId}_${definitionId}`; - } - - bucketDataV3(groupId: number, definitionId: BucketDefinitionId): mongo.Collection { - return this.db.collection(this.bucketDataCollectionNameV3(groupId, definitionId)); - } - async listBucketDataCollectionsV3(groupId?: number): Promise[]> { const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`; const collections = await this.db.listCollections({}, { nameOnly: true }).toArray(); @@ -103,25 +89,6 @@ export class PowerSyncMongo { .map((collection) => this.db.collection(collection.name)); } - bucketStateCollectionNameV3(replicationStreamId: number) { - return `bucket_state_${replicationStreamId}`; - } - - bucketStateV3(replicationStreamId: number): mongo.Collection { - return this.db.collection(this.bucketStateCollectionNameV3(replicationStreamId)); - } - - bucketParameterCollectionNameV3(replicationStreamId: number, indexId: ParameterIndexId) { - return `parameter_index_${replicationStreamId}_${indexId}`; - } - - parameterIndexV3( - replicationStreamId: number, - indexId: ParameterIndexId - ): mongo.Collection { - return this.db.collection(this.bucketParameterCollectionNameV3(replicationStreamId, indexId)); - } - /** * Not safe for user-provided prefix - only for hardcoded values. */ @@ -132,6 +99,7 @@ export class PowerSyncMongo { .filter((collection) => collection.name.startsWith(prefix)) .map((collection) => this.db.collection(collection.name)); } + /** * List all parameter index collections across all replication streams. * diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index 5df1c8494..efaefc823 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -46,7 +46,7 @@ export class MongoChecksumsV3 extends MongoChecksums { for (const [definitionId, requests] of requestsByDefinition.entries()) { const groupResults = await this.computePartialChecksumsForCollection( requests, - this.db.bucket_data_v3(this.group_id, definitionId), + this.db.bucketDataV3(this.group_id, definitionId), createV3BucketFilter ); for (const checksum of groupResults.values()) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index a3855a7c7..f65b82a88 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -96,7 +96,7 @@ export class MongoCompactorV3 extends MongoCompactor { ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { if (definitionId != null) { return { - collection: this.db.bucket_data_v3( + collection: this.db.bucketDataV3( this.group_id, definitionId ) as unknown as mongo.Collection, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index e71fc00f9..91ddf7483 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -52,7 +52,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { protected async initializeVersionStorage(): Promise { const mapping = this.mapping; for (let source of mapping.allBucketDefinitionIds()) { - const collection = this.db.bucket_data_v3(this.group_id, source).collectionName; + const collection = this.db.bucketDataV3(this.group_id, source).collectionName; await this.db.db .createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }) .catch((error) => { @@ -334,7 +334,7 @@ export async function* getBucketDataBatchV3( } })); - const cursor = ctx.db.bucket_data_v3(ctx.group_id, definitionId).find( + const cursor = ctx.db.bucketDataV3(ctx.group_id, definitionId).find( { $or: filters }, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 804f54a52..841874c3a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -260,7 +260,7 @@ export class PersistedBatchV3 extends PersistedBatch { } for (const [definitionId, documents] of operationsByDefinition.entries()) { - await this.db.bucket_data_v3(this.group_id, definitionId).bulkWrite( + await this.db.bucketDataV3(this.group_id, definitionId).bulkWrite( documents.map((document) => ({ insertOne: { document: taggedBucketDataDocumentToV3(document) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts index 3854bcff4..f4560f077 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -1,8 +1,8 @@ -import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; import { + BucketDataDocumentV3, BucketParameterDocumentV3, BucketStateDocumentV3, CommonSourceTableDocument, @@ -37,7 +37,14 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { } bucketStateV3(replicationStreamId: number): mongo.Collection { - return this.upstream.bucketStateV3(replicationStreamId); + return this.db.collection(`bucket_state_${replicationStreamId}`); + } + + parameterIndexV3( + replicationStreamId: number, + indexId: ParameterIndexId + ): mongo.Collection { + return this.db.collection(`parameter_index_${replicationStreamId}_${indexId}`); } sourceTablesV3(replicationStreamId: number): mongo.Collection { @@ -81,18 +88,14 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { ); } - bucket_data_v3(groupId: number, definitionId: BucketDefinitionId) { - return this.upstream.bucketDataV3(groupId, definitionId); + bucketDataV3(groupId: number, definitionId: BucketDefinitionId) { + return this.db.collection(`bucket_data_${groupId}_${definitionId}`); } listBucketDataCollectionsV3(groupId: number) { return this.upstream.listBucketDataCollectionsV3(groupId); } - parameterIndexV3(replicationStreamId: number, indexId: ParameterIndexId) { - return this.upstream.parameterIndexV3(replicationStreamId, indexId); - } - async listParameterIndexCollectionsV3( replicationStreamId: number ): Promise<{ collection: mongo.Collection; indexId: ParameterIndexId }[]> { diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 0b6e244da..04321e205 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -225,7 +225,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0]; expect(parameterIndexId).toBeDefined(); - const parameterEntry = await mongoFactory.db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({}); + const parameterEntry = await db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({}); expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']); }); From 6dc627980fc1988da12d6d2e96af6c85811f0cd9 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 13:34:53 +0200 Subject: [PATCH 71/93] Split v1/v3 models. --- .../src/storage/MongoBucketStorage.ts | 2 +- .../implementation/BucketDefinitionMapping.ts | 2 +- .../implementation/MongoPersistedSyncRules.ts | 2 +- .../MongoPersistedSyncRulesContent.ts | 2 +- .../implementation/common/MongoBucketBatch.ts | 2 +- .../implementation/common/MongoChecksums.ts | 2 +- .../implementation/common/MongoCompactor.ts | 8 +- .../common/MongoSyncBucketStorage.ts | 8 +- .../implementation/common/PersistedBatch.ts | 2 +- .../common/VersionedPowerSyncMongoBase.ts | 2 +- .../implementation/{ => common}/models.ts | 150 +++--------------- .../src/storage/implementation/db.ts | 14 +- .../implementation/v1/MongoCompactorV1.ts | 9 +- .../v1/MongoSyncBucketStorageV1.ts | 6 +- .../implementation/v1/PersistedBatchV1.ts | 12 +- .../implementation/v1/SourceRecordStoreV1.ts | 3 +- .../v1/VersionedPowerSyncMongoV1.ts | 9 +- .../src/storage/implementation/v1/models.ts | 64 ++++++++ .../implementation/v3/MongoCompactorV3.ts | 10 +- .../v3/MongoSyncBucketStorageV3.ts | 4 +- .../implementation/v3/PersistedBatchV3.ts | 10 +- .../implementation/v3/SourceRecordStoreV3.ts | 2 +- .../v3/VersionedPowerSyncMongoV3.ts | 4 +- .../src/storage/implementation/v3/models.ts | 67 ++++++++ .../src/storage/storage-index.ts | 4 +- .../module-mongodb-storage/src/utils/util.ts | 2 +- .../test/src/storage_sync.test.ts | 3 +- 27 files changed, 206 insertions(+), 199 deletions(-) rename modules/module-mongodb-storage/src/storage/implementation/{ => common}/models.ts (73%) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v1/models.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/models.ts diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 5669e2589..ea3843e96 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -8,7 +8,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { PowerSyncMongo } from './implementation/db.js'; -import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js'; +import { getMongoStorageConfig, SyncRuleDocument } from './implementation/common/models.js'; import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js'; import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts index 5b6a52db6..5d8947219 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -5,7 +5,7 @@ import { ParameterLookupScope, SyncConfigWithErrors } from '@powersync/service-sync-rules'; -import { SyncRuleDocument } from './models.js'; +import { SyncRuleDocument } from './common/models.js'; export type BucketDefinitionId = string; export type ParameterIndexId = string; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index ac496b55a..77d07f1f0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -11,7 +11,7 @@ import { } from '@powersync/service-sync-rules'; import { storage } from '@powersync/service-core'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { StorageConfig } from './models.js'; +import { StorageConfig } from './common/models.js'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; export class MongoPersistedSyncRules implements storage.PersistedSyncRules { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts index e503a229d..0399dbada 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts @@ -2,7 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; import { MongoSyncRulesLock } from './MongoSyncRulesLock.js'; import { PowerSyncMongo } from './db.js'; -import { getMongoStorageConfig, SyncRuleDocument } from './models.js'; +import { getMongoStorageConfig, SyncRuleDocument } from './common/models.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts index 1efb42a4a..68a3e0d26 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts @@ -27,7 +27,7 @@ import { import * as timers from 'node:timers/promises'; import { mongoTableId } from '../../../utils/util.js'; import type { VersionedPowerSyncMongo } from '../db.js'; -import { SyncRuleDocument } from '../models.js'; +import { SyncRuleDocument } from './models.js'; import { LoadedSourceRecord, SourceRecordStore } from './SourceRecordStore.js'; import { MongoIdSequence } from '../MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from '../MongoWriteCheckpointAPI.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts index 02dd6161a..1485f65f4 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts @@ -16,7 +16,7 @@ import type { VersionedPowerSyncMongo } from '../db.js'; import * as lib_mongo from '@powersync/lib-service-mongodb'; import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; -import { BucketDataDocumentBase, StorageConfig } from '../models.js'; +import { BucketDataDocumentBase, StorageConfig } from './models.js'; export interface FetchPartialBucketChecksumV3 { bucket: string; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts index 54cbc7c74..b6a956e1b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts @@ -13,15 +13,15 @@ import type { VersionedPowerSyncMongo } from '../db.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { BucketDataDocumentBase, - BucketDataDocumentV1, - BucketDataDocumentV3, LEGACY_BUCKET_DATA_DEFINITION_ID, TaggedBucketDataDocument, BucketStateDocumentBase, bucketDataDocumentToTagged -} from '../models.js'; +} from './models.js'; +import { BucketDataDocumentV1 } from '../v1/models.js'; +import { BucketDataDocumentV3 } from '../v3/models.js'; import { cacheKey } from '../OperationBatch.js'; -import { MongoSyncBucketStorage } from '../createMongoSyncBucketStorage.js'; +import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; interface CurrentBucketState { /** Bucket name */ diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts index 2b12e0f2a..84a10f345 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts @@ -28,13 +28,7 @@ import * as timers from 'timers/promises'; import { idPrefixFilter, retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import type { VersionedPowerSyncMongo } from '../db.js'; -import { - BucketDataKeyV1, - BucketStateDocument, - CommonSourceTableDocument, - SourceKey, - StorageConfig -} from '../models.js'; +import { CommonSourceTableDocument, StorageConfig } from './models.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 46bfbbae6..1832c1927 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -8,7 +8,7 @@ import { MongoIdSequence } from '../MongoIdSequence.js'; import type { VersionedPowerSyncMongo } from '../db.js'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { TaggedBucketParameterDocument, TaggedBucketDataDocument } from '../models.js'; +import { TaggedBucketParameterDocument, TaggedBucketDataDocument } from './models.js'; import { mongoTableId } from '../../../utils/util.js'; import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts index f5c911d65..bf34cc9e3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts @@ -1,6 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { PowerSyncMongo } from '../db.js'; -import { CommonSourceTableDocument, StorageConfig } from '../models.js'; +import { CommonSourceTableDocument, StorageConfig } from './models.js'; export abstract class BaseVersionedPowerSyncMongo { readonly client: mongo.MongoClient; diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/common/models.ts similarity index 73% rename from modules/module-mongodb-storage/src/storage/implementation/models.ts rename to modules/module-mongodb-storage/src/storage/implementation/common/models.ts index 9f64e8d97..b930b7eb3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/models.ts @@ -3,7 +3,9 @@ import { SqliteJsonValue } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { event_types } from '@powersync/service-types'; import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; -import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; +import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; +import type { CurrentDataDocument, SourceTableDocumentV1 } from '../v1/models.js'; +import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from '../v3/models.js'; /** * Replica id uniquely identifying a row on the source database. @@ -30,76 +32,42 @@ export interface SourceTableKey { k: ReplicaId; } -export interface BucketDataKeyV1 { - /** group_id */ - g: number; +export interface BucketDataKey { /** bucket name */ b: string; /** op_id */ o: bigint; } -export interface BucketDataKeyV3 { - /** bucket name */ - b: string; - /** op_id */ - o: bigint; -} - -export interface CurrentDataDocument { - _id: SourceKey; - data: bson.Binary; - buckets: CurrentBucket[]; - lookups: bson.Binary[]; -} - -export interface CurrentBucketV3 extends CurrentBucket { - def: BucketDefinitionId; -} - -export interface RecordedLookupV3 { - i: ParameterIndexId; - l: bson.Binary; -} - -export interface CurrentDataDocumentV3 { - _id: ReplicaId; - data: bson.Binary | null; - buckets: CurrentBucketV3[]; - lookups: RecordedLookupV3[]; - /** - * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete. - * - * This must only be set if buckets = [], lookups = []. - */ - pending_delete?: bigint; -} - export interface CurrentBucket { bucket: string; table: string; id: string; } -export interface BucketParameterDocument { +export interface BucketParameterDocumentBase { _id: bigint; - key: SourceKey; + key: TKey; lookup: bson.Binary; bucket_parameters: Record[]; } -export interface BucketParameterDocumentV3 extends Omit { - key: SourceTableKey; +export interface TaggedBucketParameterDocument extends BucketParameterDocumentBase { + index: ParameterIndexId; } -export interface TaggedBucketParameterDocument { - _id: bigint; - key: BucketParameterDocument['key'] | BucketParameterDocumentV3['key']; - lookup: bson.Binary; - bucket_parameters: Record[]; - index: ParameterIndexId; +export function bucketParameterDocumentToTagged( + document: BucketParameterDocumentBase, + index: ParameterIndexId +): TaggedBucketParameterDocument { + return { + ...document, + index + }; } +export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; + export interface BucketDataProperties { op: OpType; source_table?: bson.ObjectId; @@ -112,15 +80,7 @@ export interface BucketDataProperties { } export interface BucketDataDocumentBase extends BucketDataProperties { - _id: BucketDataKeyV3; -} - -export interface BucketDataDocumentV1 extends BucketDataDocumentBase { - _id: BucketDataKeyV1; -} - -export interface BucketDataDocumentV3 extends BucketDataDocumentBase { - _id: BucketDataKeyV3; + _id: BucketDataKey; } /** @@ -128,7 +88,7 @@ export interface BucketDataDocumentV3 extends BucketDataDocumentBase { */ export interface TaggedBucketDataDocument extends BucketDataProperties { def: BucketDefinitionId; - _id: BucketDataKeyV3; + _id: BucketDataKey; } /** @@ -141,8 +101,8 @@ export const LEGACY_BUCKET_DATA_DEFINITION_ID = '0'; */ export const LEGACY_BUCKET_PARAMETER_INDEX_ID = '0'; -export function bucketDataDocumentToTagged( - document: BucketDataDocumentV1 | BucketDataDocumentV3, +export function bucketDataDocumentToTagged( + document: TDocument, definitionId: BucketDefinitionId ): TaggedBucketDataDocument { return { @@ -155,48 +115,6 @@ export function bucketDataDocumentToTagged( }; } -export function taggedBucketDataDocumentToV1( - groupId: number, - document: TaggedBucketDataDocument -): BucketDataDocumentV1 { - const { def: _definitionId, _id: _id, ...rest } = document; - return { - _id: { - g: groupId, - b: _id.b, - o: _id.o - }, - ...rest - }; -} - -export function taggedBucketDataDocumentToV3(document: TaggedBucketDataDocument): BucketDataDocumentV3 { - const { def: _definitionId, ...rest } = document; - return rest; -} - -export function bucketParameterDocumentToTagged( - document: BucketParameterDocument | BucketParameterDocumentV3, - index: ParameterIndexId -): TaggedBucketParameterDocument { - return { - ...document, - index - }; -} - -export function taggedBucketParameterDocumentToV1(document: TaggedBucketParameterDocument): BucketParameterDocument { - const { index: _index, ...rest } = document; - return rest as BucketParameterDocument; -} - -export function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3 { - const { index: _index, ...rest } = document; - return rest as BucketParameterDocumentV3; -} - -export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR'; - export interface SourceTableDocument { _id: bson.ObjectId; connection_id: number; @@ -209,16 +127,6 @@ export interface SourceTableDocument { snapshot_status: SourceTableDocumentSnapshotStatus | undefined; } -export interface SourceTableDocumentV1 extends SourceTableDocument { - group_id: number; -} - -export interface SourceTableDocumentV3 extends SourceTableDocument { - bucket_data_source_ids: BucketDefinitionId[]; - parameter_lookup_source_ids: ParameterIndexId[]; - latest_pending_delete?: InternalOpId | undefined; -} - export interface SourceTableDocumentSnapshotStatus { total_estimated_count: number; replicated_count: number; @@ -261,20 +169,6 @@ export interface BucketStateDocumentBase { }; } -export interface BucketStateDocumentV1 extends BucketStateDocumentBase { - _id: BucketStateDocumentBase['_id'] & { - g: number; - }; -} - -export interface BucketStateDocumentV3 extends BucketStateDocumentBase { - _id: BucketStateDocumentBase['_id'] & { - d: BucketDefinitionId; - }; -} - -export type BucketStateDocument = BucketStateDocumentV1; - export interface IdSequenceDocument { _id: string; op_id: bigint; diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 104ac9b4f..022b18b80 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -5,14 +5,9 @@ import { POWERSYNC_VERSION, storage } from '@powersync/service-core'; import { MongoStorageConfig } from '../../types/types.js'; import { BaseVersionedPowerSyncMongo } from './common/VersionedPowerSyncMongoBase.js'; import { - BucketDataDocumentV1, - BucketDataDocumentV3, - BucketParameterDocument, - BucketStateDocumentV1, CheckpointEventDocument, ClientConnectionDocument, CommonSourceTableDocument, - CurrentDataDocument, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, @@ -20,7 +15,14 @@ import { StorageConfig, SyncRuleDocument, WriteCheckpointDocument -} from './models.js'; +} from './common/models.js'; +import { + BucketDataDocumentV1, + BucketParameterDocument, + BucketStateDocumentV1, + CurrentDataDocument +} from './v1/models.js'; +import { BucketDataDocumentV3 } from './v3/models.js'; import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js'; import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 986659919..d2b6407fb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -3,14 +3,11 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { BucketDataDocumentBase, - BucketDataDocumentV1, - BucketDataKeyV1, BucketStateDocumentBase, - BucketStateDocumentV1, LEGACY_BUCKET_DATA_DEFINITION_ID, - TaggedBucketDataDocument, - taggedBucketDataDocumentToV1 -} from '../models.js'; + TaggedBucketDataDocument +} from '../common/models.js'; +import { BucketDataKeyV1, BucketStateDocumentV1, taggedBucketDataDocumentToV1 } from './models.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { DirtyBucket, MongoCompactor } from '../common/MongoCompactor.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index d559e88d4..158ee548d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -27,12 +27,10 @@ import { } from '../common/MongoSyncBucketStorageContext.js'; import { bucketDataDocumentToTagged, - BucketDataDocumentV1, - BucketDataKeyV1, - BucketStateDocument, CommonSourceTableDocument, LEGACY_BUCKET_DATA_DEFINITION_ID -} from '../models.js'; +} from '../common/models.js'; +import { BucketDataDocumentV1, BucketDataKeyV1, BucketStateDocument } from './models.js'; import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; import { MongoChecksumsV1 } from './MongoChecksumsV1.js'; import { MongoCompactorV1 } from './MongoCompactorV1.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index 33f19bfc6..1a5cec510 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -11,16 +11,14 @@ import { SaveParameterDataOptions, UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; +import { LEGACY_BUCKET_DATA_DEFINITION_ID, LEGACY_BUCKET_PARAMETER_INDEX_ID, SourceKey } from '../common/models.js'; import { - BucketStateDocumentV1, BucketParameterDocument, + BucketStateDocumentV1, CurrentDataDocument, - LEGACY_BUCKET_DATA_DEFINITION_ID, - LEGACY_BUCKET_PARAMETER_INDEX_ID, - SourceKey, - taggedBucketParameterDocumentToV1, - taggedBucketDataDocumentToV1 -} from '../models.js'; + taggedBucketDataDocumentToV1, + taggedBucketParameterDocumentToV1 +} from './models.js'; import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; import { BucketStateUpdate } from '../common/PersistedBatch.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts index 42d6eafcd..170ec0a2c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts @@ -10,7 +10,8 @@ import { LoadedSourceRecord, SourceRecordStore } from '../common/SourceRecordStore.js'; -import { CurrentDataDocument, SourceKey } from '../models.js'; +import { SourceKey } from '../common/models.js'; +import { CurrentDataDocument } from './models.js'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts index 6d4ebc9e9..4d723ea42 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts @@ -1,12 +1,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; -import { - BucketDataDocumentV1, - BucketParameterDocument, - BucketStateDocumentV1, - CommonSourceTableDocument, - CurrentDataDocument -} from '../models.js'; +import { CommonSourceTableDocument } from '../common/models.js'; +import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './models.js'; export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo { get sourceRecordsV1(): mongo.Collection { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts new file mode 100644 index 000000000..463c3449b --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts @@ -0,0 +1,64 @@ +import { + BucketParameterDocumentBase, + BucketDataDocumentBase, + BucketStateDocumentBase, + SourceKey, + SourceTableDocument, + TaggedBucketDataDocument, + TaggedBucketParameterDocument +} from '../common/models.js'; +import * as bson from 'bson'; + +export interface BucketDataKeyV1 { + /** group_id */ + g: number; + /** bucket name */ + b: string; + /** op_id */ + o: bigint; +} + +export interface CurrentDataDocument { + _id: SourceKey; + data: bson.Binary; + buckets: import('../common/models.js').CurrentBucket[]; + lookups: bson.Binary[]; +} + +export interface BucketParameterDocument extends BucketParameterDocumentBase {} + +export interface BucketDataDocumentV1 extends BucketDataDocumentBase { + _id: BucketDataKeyV1; +} + +export function taggedBucketDataDocumentToV1( + groupId: number, + document: TaggedBucketDataDocument +): BucketDataDocumentV1 { + const { def: _definitionId, _id: _id, ...rest } = document; + return { + _id: { + g: groupId, + b: _id.b, + o: _id.o + }, + ...rest + }; +} + +export function taggedBucketParameterDocumentToV1(document: TaggedBucketParameterDocument): BucketParameterDocument { + const { index: _index, ...rest } = document; + return rest as BucketParameterDocument; +} + +export interface SourceTableDocumentV1 extends SourceTableDocument { + group_id: number; +} + +export interface BucketStateDocumentV1 extends BucketStateDocumentBase { + _id: BucketStateDocumentBase['_id'] & { + g: number; + }; +} + +export type BucketStateDocument = BucketStateDocumentV1; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index f65b82a88..2aeda7c9c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -3,14 +3,8 @@ import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib import { InternalOpId, storage } from '@powersync/service-core'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { DirtyBucket, MongoCompactor } from '../common/MongoCompactor.js'; -import { - BucketDataDocumentBase, - BucketDataKeyV3, - BucketStateDocumentBase, - BucketStateDocumentV3, - TaggedBucketDataDocument, - taggedBucketDataDocumentToV3 -} from '../models.js'; +import { BucketDataDocumentBase, BucketStateDocumentBase, TaggedBucketDataDocument } from '../common/models.js'; +import { BucketDataKeyV3, BucketStateDocumentV3, taggedBucketDataDocumentToV3 } from './models.js'; import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 91ddf7483..f431235bb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -14,7 +14,8 @@ import * as bson from 'bson'; import { JSONBig } from '@powersync/service-jsonbig'; import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; -import { BucketDataDocumentV3, BucketParameterDocumentV3, bucketDataDocumentToTagged } from '../models.js'; +import { bucketDataDocumentToTagged, CommonSourceTableDocument } from '../common/models.js'; +import { BucketDataDocumentV3, BucketParameterDocumentV3 } from './models.js'; import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext @@ -24,7 +25,6 @@ import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; -import { CommonSourceTableDocument } from '../models.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; import { MongoChecksums } from '../common/MongoChecksums.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 841874c3a..e0e0b1198 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -12,15 +12,15 @@ import { SaveParameterDataOptions, UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; +import { SourceTableKey } from '../common/models.js'; import { - BucketStateDocumentV3, BucketParameterDocumentV3, + BucketStateDocumentV3, CurrentDataDocumentV3, - SourceTableKey, - taggedBucketParameterDocumentToV3, + SourceTableDocumentV3, taggedBucketDataDocumentToV3, - SourceTableDocumentV3 -} from '../models.js'; + taggedBucketParameterDocumentToV3 +} from './models.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; import { BucketStateUpdate } from '../common/PersistedBatch.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts index b5a3fd2a5..db00a2f5a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts @@ -7,7 +7,7 @@ import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; import { cacheKey } from '../OperationBatch.js'; import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js'; -import { CurrentDataDocumentV3, SourceTableDocumentV3 } from '../models.js'; +import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts index f4560f077..9f72e7192 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -1,14 +1,14 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; +import { CommonSourceTableDocument } from '../common/models.js'; import { BucketDataDocumentV3, BucketParameterDocumentV3, BucketStateDocumentV3, - CommonSourceTableDocument, CurrentDataDocumentV3, SourceTableDocumentV3 -} from '../models.js'; +} from './models.js'; export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts new file mode 100644 index 000000000..e2eca4c0d --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -0,0 +1,67 @@ +import { InternalOpId } from '@powersync/service-core'; +import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; +import { + BucketDataDocumentBase, + BucketDataKey, + BucketParameterDocumentBase, + CurrentBucket, + ReplicaId, + SourceTableDocument, + SourceTableKey, + TaggedBucketDataDocument, + TaggedBucketParameterDocument, + BucketStateDocumentBase +} from '../common/models.js'; +import * as bson from 'bson'; + +export interface CurrentBucketV3 extends CurrentBucket { + def: BucketDefinitionId; +} + +export interface RecordedLookupV3 { + i: ParameterIndexId; + l: bson.Binary; +} + +export interface CurrentDataDocumentV3 { + _id: ReplicaId; + data: bson.Binary | null; + buckets: CurrentBucketV3[]; + lookups: RecordedLookupV3[]; + /** + * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete. + * + * This must only be set if buckets = [], lookups = []. + */ + pending_delete?: bigint; +} + +export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase {} + +export type BucketDataKeyV3 = BucketDataKey; + +export interface BucketDataDocumentV3 extends BucketDataDocumentBase { + _id: BucketDataKeyV3; +} + +export function taggedBucketDataDocumentToV3(document: TaggedBucketDataDocument): BucketDataDocumentV3 { + const { def: _definitionId, ...rest } = document; + return rest; +} + +export function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3 { + const { index: _index, ...rest } = document; + return rest as BucketParameterDocumentV3; +} + +export interface SourceTableDocumentV3 extends SourceTableDocument { + bucket_data_source_ids: BucketDefinitionId[]; + parameter_lookup_source_ids: ParameterIndexId[]; + latest_pending_delete?: InternalOpId | undefined; +} + +export interface BucketStateDocumentV3 extends BucketStateDocumentBase { + _id: BucketStateDocumentBase['_id'] & { + d: BucketDefinitionId; + }; +} diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index 7d29df5b0..d39976cb0 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -1,6 +1,8 @@ export * from './implementation/db.js'; export * from './implementation/BucketDefinitionMapping.js'; -export * from './implementation/models.js'; +export * from './implementation/common/models.js'; +export * from './implementation/v1/models.js'; +export * from './implementation/v3/models.js'; export * from './implementation/MongoIdSequence.js'; export * from './implementation/MongoPersistedSyncRules.js'; export * from './implementation/MongoPersistedSyncRulesContent.js'; diff --git a/modules/module-mongodb-storage/src/utils/util.ts b/modules/module-mongodb-storage/src/utils/util.ts index 2e0a1cf9c..ebd6baf83 100644 --- a/modules/module-mongodb-storage/src/utils/util.ts +++ b/modules/module-mongodb-storage/src/utils/util.ts @@ -6,7 +6,7 @@ import * as uuid from 'uuid'; import { mongo } from '@powersync/lib-service-mongodb'; import { storage, utils } from '@powersync/service-core'; import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework'; -import { TaggedBucketDataDocument } from '../storage/implementation/models.js'; +import { TaggedBucketDataDocument } from '../storage/implementation/common/models.js'; export function idPrefixFilter(prefix: Partial, rest: (keyof T)[]): mongo.Condition { let filter = { diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 04321e205..1a1215087 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -6,9 +6,10 @@ import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; +import { SyncRuleDocument } from '../../src/storage/implementation/common/models.js'; import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js'; +import { CurrentBucketV3 } from '../../src/storage/implementation/v3/models.js'; import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; -import { CurrentBucketV3, SyncRuleDocument } from '../../src/storage/implementation/models.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) { From 1c0ca4734d4637c5acdcc7dd1952ce02a83e8e99 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 13:40:08 +0200 Subject: [PATCH 72/93] Move files back to their original location. --- .../src/storage/MongoBucketStorage.ts | 4 ++-- .../implementation/BucketDefinitionMapping.ts | 2 +- .../{common => }/MongoBucketBatch.ts | 18 +++++++++--------- .../{common => }/MongoChecksums.ts | 4 ++-- .../{common => }/MongoCompactor.ts | 10 +++++----- .../{common => }/MongoParameterCompactor.ts | 2 +- .../implementation/MongoPersistedSyncRules.ts | 2 +- .../MongoPersistedSyncRulesContent.ts | 2 +- .../{common => }/MongoSyncBucketStorage.ts | 12 ++++++------ .../implementation/common/PersistedBatch.ts | 2 +- .../common/VersionedPowerSyncMongoBase.ts | 2 +- .../createMongoSyncBucketStorage.ts | 4 ++-- .../src/storage/implementation/db.ts | 2 +- .../implementation/{common => }/models.ts | 6 +++--- .../implementation/v1/MongoBucketBatchV1.ts | 2 +- .../implementation/v1/MongoChecksumsV1.ts | 4 ++-- .../implementation/v1/MongoCompactorV1.ts | 4 ++-- .../v1/MongoParameterCompactorV1.ts | 2 +- .../v1/MongoSyncBucketStorageV1.ts | 16 ++++++---------- .../implementation/v1/PersistedBatchV1.ts | 2 +- .../implementation/v1/SourceRecordStoreV1.ts | 2 +- .../v1/VersionedPowerSyncMongoV1.ts | 2 +- .../src/storage/implementation/v1/models.ts | 4 ++-- .../implementation/v3/MongoBucketBatchV3.ts | 2 +- .../implementation/v3/MongoChecksumsV3.ts | 4 ++-- .../implementation/v3/MongoCompactorV3.ts | 4 ++-- .../v3/MongoParameterCompactorV3.ts | 2 +- .../v3/MongoSyncBucketStorageV3.ts | 12 ++++++------ .../implementation/v3/PersistedBatchV3.ts | 2 +- .../v3/VersionedPowerSyncMongoV3.ts | 2 +- .../src/storage/implementation/v3/models.ts | 2 +- .../src/storage/storage-index.ts | 2 +- .../module-mongodb-storage/src/utils/util.ts | 2 +- .../test/src/storage_sync.test.ts | 4 ++-- 34 files changed, 72 insertions(+), 76 deletions(-) rename modules/module-mongodb-storage/src/storage/implementation/{common => }/MongoBucketBatch.ts (98%) rename modules/module-mongodb-storage/src/storage/implementation/{common => }/MongoChecksums.ts (98%) rename modules/module-mongodb-storage/src/storage/implementation/{common => }/MongoCompactor.ts (98%) rename modules/module-mongodb-storage/src/storage/implementation/{common => }/MongoParameterCompactor.ts (98%) rename modules/module-mongodb-storage/src/storage/implementation/{common => }/MongoSyncBucketStorage.ts (98%) rename modules/module-mongodb-storage/src/storage/implementation/{common => }/models.ts (98%) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index ea3843e96..de32da4fa 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -8,12 +8,12 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { PowerSyncMongo } from './implementation/db.js'; -import { getMongoStorageConfig, SyncRuleDocument } from './implementation/common/models.js'; +import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js'; import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js'; import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; import { generateSlotName } from '../utils/util.js'; -import { MongoChecksumOptions } from './implementation/common/MongoChecksums.js'; +import { MongoChecksumOptions } from './implementation/MongoChecksums.js'; import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; export interface MongoBucketStorageOptions { diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts index 5d8947219..5b6a52db6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -5,7 +5,7 @@ import { ParameterLookupScope, SyncConfigWithErrors } from '@powersync/service-sync-rules'; -import { SyncRuleDocument } from './common/models.js'; +import { SyncRuleDocument } from './models.js'; export type BucketDefinitionId = string; export type ParameterIndexId = string; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts similarity index 98% rename from modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts rename to modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 68a3e0d26..9807dd92d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -25,16 +25,16 @@ import { utils } from '@powersync/service-core'; import * as timers from 'node:timers/promises'; -import { mongoTableId } from '../../../utils/util.js'; -import type { VersionedPowerSyncMongo } from '../db.js'; +import { mongoTableId } from '../../utils/util.js'; +import type { VersionedPowerSyncMongo } from './db.js'; import { SyncRuleDocument } from './models.js'; -import { LoadedSourceRecord, SourceRecordStore } from './SourceRecordStore.js'; -import { MongoIdSequence } from '../MongoIdSequence.js'; -import { batchCreateCustomWriteCheckpoints } from '../MongoWriteCheckpointAPI.js'; -import { OperationBatch, RecordOperation } from '../OperationBatch.js'; -import { PersistedBatch } from './PersistedBatch.js'; -import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; -import { MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; +import { LoadedSourceRecord, SourceRecordStore } from './common/SourceRecordStore.js'; +import { MongoIdSequence } from './MongoIdSequence.js'; +import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; +import { OperationBatch, RecordOperation } from './OperationBatch.js'; +import { PersistedBatch } from './common/PersistedBatch.js'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; // Currently, we can only have a single flush() at a time, since it locks the op_id sequence. // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts similarity index 98% rename from modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts rename to modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index 1485f65f4..6a81ce95c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -12,10 +12,10 @@ import { PartialChecksumMap, PartialOrFullChecksum } from '@powersync/service-core'; -import type { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from './db.js'; import * as lib_mongo from '@powersync/lib-service-mongodb'; -import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { BucketDefinitionId, BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { BucketDataDocumentBase, StorageConfig } from './models.js'; export interface FetchPartialBucketChecksumV3 { diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts similarity index 98% rename from modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts rename to modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index b6a956e1b..2bb6bbcee 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -9,8 +9,8 @@ import { utils } from '@powersync/service-core'; -import type { VersionedPowerSyncMongo } from '../db.js'; -import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import type { VersionedPowerSyncMongo } from './db.js'; +import { BucketDefinitionId } from './BucketDefinitionMapping.js'; import { BucketDataDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID, @@ -18,9 +18,9 @@ import { BucketStateDocumentBase, bucketDataDocumentToTagged } from './models.js'; -import { BucketDataDocumentV1 } from '../v1/models.js'; -import { BucketDataDocumentV3 } from '../v3/models.js'; -import { cacheKey } from '../OperationBatch.js'; +import { BucketDataDocumentV1 } from './v1/models.js'; +import { BucketDataDocumentV3 } from './v3/models.js'; +import { cacheKey } from './OperationBatch.js'; import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; interface CurrentBucketState { diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts similarity index 98% rename from modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts rename to modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts index e2a8be7ef..c7bf5342a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoParameterCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoParameterCompactor.ts @@ -2,7 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { logger } from '@powersync/lib-services-framework'; import { bson, CompactOptions, InternalOpId } from '@powersync/service-core'; import { LRUCache } from 'lru-cache'; -import type { VersionedPowerSyncMongo } from '../db.js'; +import type { VersionedPowerSyncMongo } from './db.js'; type ParameterCompactionReadDocument = { _id: InternalOpId; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index 77d07f1f0..ac496b55a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -11,7 +11,7 @@ import { } from '@powersync/service-sync-rules'; import { storage } from '@powersync/service-core'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { StorageConfig } from './common/models.js'; +import { StorageConfig } from './models.js'; import { ServiceAssertionError } from '@powersync/lib-services-framework'; export class MongoPersistedSyncRules implements storage.PersistedSyncRules { diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts index 0399dbada..e503a229d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts @@ -2,7 +2,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; import { MongoSyncRulesLock } from './MongoSyncRulesLock.js'; import { PowerSyncMongo } from './db.js'; -import { getMongoStorageConfig, SyncRuleDocument } from './common/models.js'; +import { getMongoStorageConfig, SyncRuleDocument } from './models.js'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts similarity index 98% rename from modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts rename to modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 84a10f345..cfd45c959 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -25,16 +25,16 @@ import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powers import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; -import { idPrefixFilter, retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; -import { MongoBucketStorage } from '../../MongoBucketStorage.js'; -import type { VersionedPowerSyncMongo } from '../db.js'; +import { idPrefixFilter, retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; +import { MongoBucketStorage } from '../MongoBucketStorage.js'; +import type { VersionedPowerSyncMongo } from './db.js'; import { CommonSourceTableDocument, StorageConfig } from './models.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; -import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; -import { MongoWriteCheckpointAPI } from '../MongoWriteCheckpointAPI.js'; -import { MongoSyncBucketStorageContext } from './MongoSyncBucketStorageContext.js'; +import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; +import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; +import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js'; import { MongoBucketBatchOptions } from './MongoBucketBatch.js'; export interface MongoSyncBucketStorageOptions { diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 1832c1927..46bfbbae6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -8,7 +8,7 @@ import { MongoIdSequence } from '../MongoIdSequence.js'; import type { VersionedPowerSyncMongo } from '../db.js'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { TaggedBucketParameterDocument, TaggedBucketDataDocument } from './models.js'; +import { TaggedBucketParameterDocument, TaggedBucketDataDocument } from '../models.js'; import { mongoTableId } from '../../../utils/util.js'; import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts index bf34cc9e3..f5c911d65 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts @@ -1,6 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { PowerSyncMongo } from '../db.js'; -import { CommonSourceTableDocument, StorageConfig } from './models.js'; +import { CommonSourceTableDocument, StorageConfig } from '../models.js'; export abstract class BaseVersionedPowerSyncMongo { readonly client: mongo.MongoClient; diff --git a/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts index 712f2f7d4..569a5eb58 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts @@ -1,11 +1,11 @@ import { storage } from '@powersync/service-core'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; -import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorage.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js'; import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js'; import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js'; -export { MongoSyncBucketStorageOptions } from './common/MongoSyncBucketStorage.js'; +export { MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js'; export type { MongoSyncBucketStorage }; diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 022b18b80..91da6fa2c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -15,7 +15,7 @@ import { StorageConfig, SyncRuleDocument, WriteCheckpointDocument -} from './common/models.js'; +} from './models.js'; import { BucketDataDocumentV1, BucketParameterDocument, diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts similarity index 98% rename from modules/module-mongodb-storage/src/storage/implementation/common/models.ts rename to modules/module-mongodb-storage/src/storage/implementation/models.ts index b930b7eb3..a0c9b188a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -3,9 +3,9 @@ import { SqliteJsonValue } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { event_types } from '@powersync/service-types'; import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; -import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; -import type { CurrentDataDocument, SourceTableDocumentV1 } from '../v1/models.js'; -import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from '../v3/models.js'; +import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; +import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js'; +import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js'; /** * Replica id uniquely identifying a row on the source database. diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts index 277a2cea8..b8b6f70cd 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts @@ -1,4 +1,4 @@ -import { MongoBucketBatch, MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; +import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js'; import { SourceRecordStore } from '../common/SourceRecordStore.js'; import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts index 894af7115..a8c8a6a38 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts @@ -5,8 +5,8 @@ import { InternalOpId, PartialChecksumMap } from '@powersync/service-core'; -import { FetchPartialBucketChecksumByBucket } from '../common/MongoChecksums.js'; -import { MongoChecksums } from '../common/MongoChecksums.js'; +import { FetchPartialBucketChecksumByBucket } from '../MongoChecksums.js'; +import { MongoChecksums } from '../MongoChecksums.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoChecksumsV1 extends MongoChecksums { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index d2b6407fb..44dca3f76 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -6,10 +6,10 @@ import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID, TaggedBucketDataDocument -} from '../common/models.js'; +} from '../models.js'; import { BucketDataKeyV1, BucketStateDocumentV1, taggedBucketDataDocumentToV1 } from './models.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { DirtyBucket, MongoCompactor } from '../common/MongoCompactor.js'; +import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts index f7053ea64..4ff5f4cb6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoParameterCompactorV1.ts @@ -1,5 +1,5 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoParameterCompactorV1 extends MongoParameterCompactor { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 158ee548d..3d7cfa43c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -15,21 +15,17 @@ import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-ru import * as bson from 'bson'; import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; -import { MongoChecksums } from '../common/MongoChecksums.js'; -import { MongoCompactOptions, MongoCompactor } from '../common/MongoCompactor.js'; -import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; +import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; -import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; -import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorage.js'; +import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js'; -import { - bucketDataDocumentToTagged, - CommonSourceTableDocument, - LEGACY_BUCKET_DATA_DEFINITION_ID -} from '../common/models.js'; +import { bucketDataDocumentToTagged, CommonSourceTableDocument, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js'; import { BucketDataDocumentV1, BucketDataKeyV1, BucketStateDocument } from './models.js'; import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; import { MongoChecksumsV1 } from './MongoChecksumsV1.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index 1a5cec510..461f2c29a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -11,7 +11,7 @@ import { SaveParameterDataOptions, UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; -import { LEGACY_BUCKET_DATA_DEFINITION_ID, LEGACY_BUCKET_PARAMETER_INDEX_ID, SourceKey } from '../common/models.js'; +import { LEGACY_BUCKET_DATA_DEFINITION_ID, LEGACY_BUCKET_PARAMETER_INDEX_ID, SourceKey } from '../models.js'; import { BucketParameterDocument, BucketStateDocumentV1, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts index 170ec0a2c..fcd1af1ef 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts @@ -10,7 +10,7 @@ import { LoadedSourceRecord, SourceRecordStore } from '../common/SourceRecordStore.js'; -import { SourceKey } from '../common/models.js'; +import { SourceKey } from '../models.js'; import { CurrentDataDocument } from './models.js'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts index 4d723ea42..15a99f85e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts @@ -1,6 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; -import { CommonSourceTableDocument } from '../common/models.js'; +import { CommonSourceTableDocument } from '../models.js'; import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './models.js'; export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts index 463c3449b..eecdbc559 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts @@ -6,7 +6,7 @@ import { SourceTableDocument, TaggedBucketDataDocument, TaggedBucketParameterDocument -} from '../common/models.js'; +} from '../models.js'; import * as bson from 'bson'; export interface BucketDataKeyV1 { @@ -21,7 +21,7 @@ export interface BucketDataKeyV1 { export interface CurrentDataDocument { _id: SourceKey; data: bson.Binary; - buckets: import('../common/models.js').CurrentBucket[]; + buckets: import('../models.js').CurrentBucket[]; lookups: bson.Binary[]; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts index 9d0af9e5e..df74fc399 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -1,6 +1,6 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; -import { MongoBucketBatch, MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; +import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js'; import { SourceRecordStore } from '../common/SourceRecordStore.js'; import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index efaefc823..590c7348b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -11,8 +11,8 @@ import { emptyChecksumForRequest, FetchPartialBucketChecksumV3, MongoChecksumOptions -} from '../common/MongoChecksums.js'; -import { MongoChecksums } from '../common/MongoChecksums.js'; +} from '../MongoChecksums.js'; +import { MongoChecksums } from '../MongoChecksums.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoChecksumsV3 extends MongoChecksums { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 2aeda7c9c..3b800e896 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -2,8 +2,8 @@ import { MONGO_OPERATION_TIMEOUT_MS, mongo } from '@powersync/lib-service-mongod import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { DirtyBucket, MongoCompactor } from '../common/MongoCompactor.js'; -import { BucketDataDocumentBase, BucketStateDocumentBase, TaggedBucketDataDocument } from '../common/models.js'; +import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { BucketDataDocumentBase, BucketStateDocumentBase, TaggedBucketDataDocument } from '../models.js'; import { BucketDataKeyV3, BucketStateDocumentV3, taggedBucketDataDocumentToV3 } from './models.js'; import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts index ecb935a35..433c0ac7b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterCompactorV3.ts @@ -1,5 +1,5 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoParameterCompactorV3 extends MongoParameterCompactor { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index f431235bb..3978f6c14 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -14,22 +14,22 @@ import * as bson from 'bson'; import { JSONBig } from '@powersync/service-jsonbig'; import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; -import { bucketDataDocumentToTagged, CommonSourceTableDocument } from '../common/models.js'; +import { bucketDataDocumentToTagged, CommonSourceTableDocument } from '../models.js'; import { BucketDataDocumentV3, BucketParameterDocumentV3 } from './models.js'; import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js'; -import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../common/MongoSyncBucketStorage.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; -import { MongoBucketBatchOptions } from '../common/MongoBucketBatch.js'; +import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; -import { MongoChecksums } from '../common/MongoChecksums.js'; -import { MongoCompactOptions, MongoCompactor } from '../common/MongoCompactor.js'; -import { MongoParameterCompactor } from '../common/MongoParameterCompactor.js'; +import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoCompactorV3 } from './MongoCompactorV3.js'; import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index e0e0b1198..17049e9ca 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -12,7 +12,7 @@ import { SaveParameterDataOptions, UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; -import { SourceTableKey } from '../common/models.js'; +import { SourceTableKey } from '../models.js'; import { BucketParameterDocumentV3, BucketStateDocumentV3, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts index 9f72e7192..a6fabdd6c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -1,7 +1,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; -import { CommonSourceTableDocument } from '../common/models.js'; +import { CommonSourceTableDocument } from '../models.js'; import { BucketDataDocumentV3, BucketParameterDocumentV3, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index e2eca4c0d..5648540f3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -11,7 +11,7 @@ import { TaggedBucketDataDocument, TaggedBucketParameterDocument, BucketStateDocumentBase -} from '../common/models.js'; +} from '../models.js'; import * as bson from 'bson'; export interface CurrentBucketV3 extends CurrentBucket { diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index d39976cb0..bb0a5de62 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -1,6 +1,6 @@ export * from './implementation/db.js'; export * from './implementation/BucketDefinitionMapping.js'; -export * from './implementation/common/models.js'; +export * from './implementation/models.js'; export * from './implementation/v1/models.js'; export * from './implementation/v3/models.js'; export * from './implementation/MongoIdSequence.js'; diff --git a/modules/module-mongodb-storage/src/utils/util.ts b/modules/module-mongodb-storage/src/utils/util.ts index ebd6baf83..2e0a1cf9c 100644 --- a/modules/module-mongodb-storage/src/utils/util.ts +++ b/modules/module-mongodb-storage/src/utils/util.ts @@ -6,7 +6,7 @@ import * as uuid from 'uuid'; import { mongo } from '@powersync/lib-service-mongodb'; import { storage, utils } from '@powersync/service-core'; import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework'; -import { TaggedBucketDataDocument } from '../storage/implementation/common/models.js'; +import { TaggedBucketDataDocument } from '../storage/implementation/models.js'; export function idPrefixFilter(prefix: Partial, rest: (keyof T)[]): mongo.Condition { let filter = { diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 1a1215087..3b4913d03 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -6,7 +6,7 @@ import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; -import { SyncRuleDocument } from '../../src/storage/implementation/common/models.js'; +import { SyncRuleDocument } from '../../src/storage/implementation/models.js'; import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js'; import { CurrentBucketV3 } from '../../src/storage/implementation/v3/models.js'; import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; @@ -224,7 +224,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const ruleMapping: SyncRuleDocument['rule_mapping'] | undefined = syncRule?.rule_mapping; expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0); - const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0]; + const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined; expect(parameterIndexId).toBeDefined(); const parameterEntry = await db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({}); expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']); From cc37c9dbf577da4782c62051a3c2513fc2094a09 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 13:49:17 +0200 Subject: [PATCH 73/93] Update docs. --- docs/{data-ownership.md => storage-v3.md} | 36 ++++++++++++++--------- 1 file changed, 22 insertions(+), 14 deletions(-) rename docs/{data-ownership.md => storage-v3.md} (68%) diff --git a/docs/data-ownership.md b/docs/storage-v3.md similarity index 68% rename from docs/data-ownership.md rename to docs/storage-v3.md index c5e4e3ca7..8381d4be8 100644 --- a/docs/data-ownership.md +++ b/docs/storage-v3.md @@ -1,4 +1,4 @@ -# Storage version 3 - Data structure and "ownership" +# Storage version 3 - Data structure ## Replication stream @@ -21,21 +21,21 @@ It is possible to have multiple replication streams running concurrently, for ex ## source_table -Belongs to a replication stream. +Scoped to a replication stream. -Scope: `source_table_${stream_id}` +Collection: `source_table_${stream_id}` -[FUTURE CHANGE] May have multiple copies per table per stream, especially when adding definitions. +[FUTURE CHANGE] May have multiple copies per phyisical table per stream, especially when adding definitions to a stream. [FUTURE CHANGE] We can remove a source definition from a source table, but never add one. ## source_records (previously current_data) -Owned by source table. +Scoped to a source_table in a replication stream. -Scope: `source_records_${stream_id}_${source_table_id}` +Collection: `source_records_${stream_id}_${source_table_id}` -The `_id` field, is the source row id. Unlike V1 storage model, this does not include `g` (group_id) or `t` (table id), since those are already encapsulated in the collection name. +The `_id` field is now the source row id. Unlike V1 storage model, this does not include `g` (group_id) or `t` (table id), since those are already encapsulated in the collection name. When a table is dropped, we first create relevant REMOVE operations, then drop the relevant current_data collection. @@ -49,22 +49,30 @@ When all definitions for a source table is removed, we remove the drop the corre ## bucket_data -Owned by replication stream. +Scoped by replication stream and definition id. -Scoped by definition. +Collection: `bucket_data_${stream_id}_${definition_id}` -Scope: `bucket_data_${stream_id}_${definition_id}` +`_id.g` is removed, since this is encapsulated in the collection name now. + +`definition_id` is new here - that is not tracked in storage V1. [FUTURE CHANGE] collection must be dropped when the definition is removed. ## parameter_index (previously bucket_parameters) -Owned by replication stream. - -Scoped by definition. +Scoped by replication stream and index definition. -Scope: `parameter_index_${stream_id}_${index_id}` +Collection: `parameter_index_${stream_id}_${index_id}` _Also_ indexed by compound `key`, which includes {t: source_table_id, k: source_record_key} The `lookup` array drops the first two fields compared to V1 lookups (lookupName and queryId), since those are encapsulated in `index_id` in the collection name. In-memory, we use lookupName = indexId, queryId = '' (may change in the future). + +## bucket_state + +Scoped by replication stream. + +Collection: `bucket_state_${stream_id}`. + +`_id` is now compound: `{d: , b: }` (previously `{g, b}`) From 55d6a682ac5163ce654ea436054e979ccb27fb4f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 13:59:51 +0200 Subject: [PATCH 74/93] Add back clear for current_data. --- .../v1/MongoSyncBucketStorageV1.ts | 21 +++++++++- .../test/src/storage_sync.test.ts | 38 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 3d7cfa43c..7e92fc87e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -25,7 +25,12 @@ import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js'; -import { bucketDataDocumentToTagged, CommonSourceTableDocument, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js'; +import { + bucketDataDocumentToTagged, + CommonSourceTableDocument, + LEGACY_BUCKET_DATA_DEFINITION_ID, + SourceKey +} from '../models.js'; import { BucketDataDocumentV1, BucketDataKeyV1, BucketStateDocument } from './models.js'; import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; import { MongoChecksumsV1 } from './MongoChecksumsV1.js'; @@ -137,7 +142,19 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { ); } - protected async clearSourceRecords(_signal?: AbortSignal): Promise {} + protected async clearSourceRecords(signal?: AbortSignal): Promise { + await this.clearDeleteMany( + 'source records', + () => + this.db.sourceRecordsV1.deleteMany( + { + _id: idPrefixFilter({ g: this.group_id }, ['t', 'k']) + }, + { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS } + ), + signal + ); + } protected async clearBucketState(signal?: AbortSignal): Promise { await this.clearDeleteMany( diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index 3b4913d03..a72668161 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -269,6 +269,44 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor expect(sourceRecordCollections).toEqual([]); }); + test.runIf(storageVersion < 3)('clear removes v1 current_data rows', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM test + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'clear-check', + description: 'shape' + }, + afterReplicaId: test_utils.rid('clear-check') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + + const mongoFactory = factory as MongoBucketStorage; + expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1); + + await bucketStorage.clear(); + + expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(0); + }); + test.runIf(storageVersion >= 3)( 'loads parameter checkpoint changes across all v3 parameter index collections', async () => { From f6b2a6f8fc20f3be59ff3193dea00f9cc15fc2b8 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 14:02:59 +0200 Subject: [PATCH 75/93] Add back missing metric tracking. --- .../src/storage/MongoBucketStorage.ts | 19 +++++++--- .../test/src/storage_sync.test.ts | 38 +++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index de32da4fa..118e48c35 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -347,7 +347,6 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { .toArray() .catch(ignoreNotExisting); - // FIXME: Handle v1 metrics const v3_parameter_aggregates = await Promise.all( (await this.db.listAllParameterIndexCollectionsV3()).map((collection) => collection @@ -363,6 +362,17 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ) ); + const v1_source_record_aggregate = await this.db.current_data + .aggregate([ + { + $collStats: { + storageStats: {} + } + } + ]) + .toArray() + .catch(ignoreNotExisting); + const source_record_aggregates = await Promise.all( (await this.db.listAllSourceRecordCollectionsV3()).map((collection) => collection @@ -384,10 +394,9 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size) + v3_parameter_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0), - replication_size_bytes: source_record_aggregates.reduce( - (total, aggregate) => total + Number(aggregate[0]?.storageStats?.size ?? 0), - 0 - ) + replication_size_bytes: + Number(v1_source_record_aggregate[0]?.storageStats?.size ?? 0) + + source_record_aggregates.reduce((total, aggregate) => total + Number(aggregate[0]?.storageStats?.size ?? 0), 0) }; } diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index a72668161..df74fe6c8 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -307,6 +307,44 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(0); }); + test.runIf(storageVersion < 3)('storage metrics include v1 current_data', async () => { + await using factory = await storageConfig.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` + bucket_definitions: + global: + data: + - SELECT id, description FROM test + `, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules); + const metricsBefore = await factory.getStorageMetrics(); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: 'metric-check', + description: 'shape' + }, + afterReplicaId: test_utils.rid('metric-check') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + + const mongoFactory = factory as MongoBucketStorage; + expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1); + + const metricsAfter = await factory.getStorageMetrics(); + expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes); + }); + test.runIf(storageVersion >= 3)( 'loads parameter checkpoint changes across all v3 parameter index collections', async () => { From 2f632d8fcdee354f09958f58790b4ced3c6291a2 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 14:09:07 +0200 Subject: [PATCH 76/93] Check if collection exists before getting storage stats. --- .../src/storage/MongoBucketStorage.ts | 54 +++++++++---------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 118e48c35..1a22afb7a 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -311,16 +311,30 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { replication_size_bytes: 0 }; } - const operations_aggregate = await this.db.bucket_data - .aggregate([ - { - $collStats: { - storageStats: {} + + const aggregateStaticCollection = async (collection: mongo.Collection) => { + // We check whether the collection exists before getting the statistics. This avoids repeated + // errors in the MongoDB logs if the collection hasn't been created yet. + const exists = + (await this.db.db.listCollections({ name: collection.collectionName }, { nameOnly: true }).toArray()).length > + 0; + if (!exists) { + return [{ storageStats: { size: 0 } }]; + } + + return collection + .aggregate([ + { + $collStats: { + storageStats: {} + } } - } - ]) - .toArray() - .catch(ignoreNotExisting); + ]) + .toArray() + .catch(ignoreNotExisting); + }; + + const operations_aggregate = await aggregateStaticCollection(this.db.bucket_data); const v3_operation_aggregates = await Promise.all( (await this.db.listBucketDataCollectionsV3()).map((collection) => collection @@ -336,16 +350,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ) ); - const parameters_aggregate = await this.db.bucket_parameters - .aggregate([ - { - $collStats: { - storageStats: {} - } - } - ]) - .toArray() - .catch(ignoreNotExisting); + const parameters_aggregate = await aggregateStaticCollection(this.db.bucket_parameters); const v3_parameter_aggregates = await Promise.all( (await this.db.listAllParameterIndexCollectionsV3()).map((collection) => @@ -362,16 +367,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { ) ); - const v1_source_record_aggregate = await this.db.current_data - .aggregate([ - { - $collStats: { - storageStats: {} - } - } - ]) - .toArray() - .catch(ignoreNotExisting); + const v1_source_record_aggregate = await aggregateStaticCollection(this.db.current_data); const source_record_aggregates = await Promise.all( (await this.db.listAllSourceRecordCollectionsV3()).map((collection) => From 29073cfe94271e1d85ef73fe0e71cb030536775a Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 14:10:16 +0200 Subject: [PATCH 77/93] Report metrics even if there are no active replication streams yet. --- .../src/storage/MongoBucketStorage.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 1a22afb7a..46a1697ef 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -303,14 +303,8 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { } }; - const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' }); - if (active_sync_rules == null) { - return { - operations_size_bytes: 0, - parameters_size_bytes: 0, - replication_size_bytes: 0 - }; - } + // For now, we get storage metrics over all v1 and v3 collections. + // In the future, we may split these metrics to report separately for active replication streams versus processing streams. const aggregateStaticCollection = async (collection: mongo.Collection) => { // We check whether the collection exists before getting the statistics. This avoids repeated From d053aef0a9f5e95bdae05c9c99bf377698a0d507 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 14:29:51 +0200 Subject: [PATCH 78/93] Update comment. --- .../src/storage/implementation/MongoCompactor.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index 2bb6bbcee..ad89bafab 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -372,6 +372,7 @@ export abstract class MongoCompactor { $lt: upperBound }, // Workaround for a clustered collection bug where the $lt operator may include upperBound. + // Technically only needed for storage V3. // https://jira.mongodb.org/browse/SERVER-121822 '_id.o': { $lt: upperBound.o } } From af75b2d8163f9ddb265c66192ddfc6d8bb91810f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 14:52:36 +0200 Subject: [PATCH 79/93] Some collection listing cleanup. --- .../src/storage/implementation/db.ts | 5 ++++- .../implementation/v3/VersionedPowerSyncMongoV3.ts | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index 91da6fa2c..dc6458ab9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -82,9 +82,12 @@ export class PowerSyncMongo { return new VersionedPowerSyncMongoV1(this, storageConfig); } + /** + * Not safe for user-provided prefix - only for hardcoded values. + */ async listBucketDataCollectionsV3(groupId?: number): Promise[]> { const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`; - const collections = await this.db.listCollections({}, { nameOnly: true }).toArray(); + const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray(); return collections .filter((collection) => collection.name.startsWith(prefix)) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts index a6fabdd6c..be1da41ac 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -88,12 +88,12 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { ); } - bucketDataV3(groupId: number, definitionId: BucketDefinitionId) { - return this.db.collection(`bucket_data_${groupId}_${definitionId}`); + bucketDataV3(replicationStreamId: number, definitionId: BucketDefinitionId) { + return this.db.collection(`bucket_data_${replicationStreamId}_${definitionId}`); } - listBucketDataCollectionsV3(groupId: number) { - return this.upstream.listBucketDataCollectionsV3(groupId); + listBucketDataCollectionsV3(replicationStreamId: number) { + return this.upstream.listBucketDataCollectionsV3(replicationStreamId); } async listParameterIndexCollectionsV3( From a1778f89c15201afa1725f8513a6f1d6bc7bcc17 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 15:05:35 +0200 Subject: [PATCH 80/93] Optimize bucket collection lookup for v3 compacting. --- .../src/storage/implementation/models.ts | 10 +++-- .../implementation/v3/MongoCompactorV3.ts | 43 +++++++++---------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index a0c9b188a..9dd0ed083 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -136,12 +136,14 @@ export interface SourceTableDocumentSnapshotStatus { /** * Record the state of each bucket. * - * Right now, this is just used to track when buckets are updated, for efficient incremental sync. - * In the future, this could be used to track operation counts, both for diagnostic purposes, and for - * determining when a compact and/or defragment could be beneficial. + * The primary use case is to track when buckets are updated, for efficient incremental sync. * - * Note: There is currently no migration to populate this collection from existing data - it is only + * The secondary use case is to track operation counts to determine whether or not a bucket should be compacted. + * + * Note: For storage V1, there is no migration to populate this collection from existing data - it is only * populated by new updates. + * + * For storage V3, these will always be present. */ export interface BucketStateDocumentBase { _id: { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 3b800e896..c30f969ce 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -88,32 +88,29 @@ export class MongoCompactorV3 extends MongoCompactor { bucket: string, definitionId: BucketDefinitionId | null ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { - if (definitionId != null) { - return { - collection: this.db.bucketDataV3( - this.group_id, - definitionId - ) as unknown as mongo.Collection, - definitionId - }; - } - - // FIXME: This is slow. It is only used when compacting a single bucket without a known definition id. - for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) { - const existing = await collection.findOne( - { '_id.b': bucket }, - { projection: { _id: 1 }, maxTimeMS: MONGO_OPERATION_TIMEOUT_MS } - ); - if (existing != null) { - const resolvedDefinitionId = collection.collectionName.replace(`bucket_data_${this.group_id}_`, ''); - return { - collection: collection as unknown as mongo.Collection, - definitionId: resolvedDefinitionId - }; + if (definitionId == null) { + // Not the _most_ efficient approach, but this is not used often + const allDefinitionIds = this.storage.mapping.allBucketDefinitionIds(); + if (allDefinitionIds.length == 0) { + return null; + } + const potentialIds = allDefinitionIds.map((definitionId) => ({ d: definitionId, b: bucket })); + const bucketState = await this.db.bucketStateV3(this.group_id).findOne({ + _id: { $in: potentialIds } + }); + if (bucketState == null) { + return null; } + definitionId = bucketState._id.d; } - return null; + return { + collection: this.db.bucketDataV3( + this.group_id, + definitionId + ) as unknown as mongo.Collection, + definitionId + }; } protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase { From f5fa3f28f5b82f07ebae5cd57de289831679a43d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 15:18:16 +0200 Subject: [PATCH 81/93] Remove unused functions. --- .../src/storage/implementation/models.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 9dd0ed083..f4af4a1e0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -351,11 +351,3 @@ export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocu export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3; export type CommonCurrentLookup = bson.Binary | RecordedLookupV3; export type CommonSourceTableDocument = SourceTableDocumentV1 | SourceTableDocumentV3; - -export function isCurrentBucketV3(bucket: CommonCurrentBucket): bucket is CurrentBucketV3 { - return 'def' in bucket; -} - -export function isRecordedLookupV3(lookup: CommonCurrentLookup): lookup is RecordedLookupV3 { - return typeof lookup === 'object' && lookup != null && 'i' in lookup && 'l' in lookup; -} From f7fd9065de422dbe1a8c349c2af05181ed1afb5d Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Wed, 1 Apr 2026 15:45:59 +0200 Subject: [PATCH 82/93] Organize imports. --- .../src/storage/MongoBucketStorage.ts | 12 +++++------ .../implementation/BucketDefinitionMapping.ts | 7 +------ .../implementation/MongoBucketBatch.ts | 9 ++++----- .../storage/implementation/MongoCompactor.ts | 12 +++++------ .../implementation/MongoPersistedSyncRules.ts | 4 ++-- .../MongoPersistedSyncRulesContent.ts | 4 ++-- .../implementation/MongoSyncBucketStorage.ts | 6 +++--- .../common/MongoSyncBucketStorageContext.ts | 2 +- .../implementation/common/PersistedBatch.ts | 7 +++---- .../src/storage/implementation/db.ts | 2 +- .../src/storage/implementation/models.ts | 4 ++-- .../implementation/v1/MongoBucketBatchV1.ts | 4 ++-- .../implementation/v1/MongoChecksumsV1.ts | 3 +-- .../implementation/v1/MongoCompactorV1.ts | 6 +++--- .../v1/MongoSyncBucketStorageV1.ts | 12 +++++------ .../implementation/v1/PersistedBatchV1.ts | 8 ++++---- .../implementation/v1/SourceRecordStoreV1.ts | 6 +++--- .../src/storage/implementation/v1/models.ts | 4 ++-- .../implementation/v3/MongoBucketBatchV3.ts | 6 +++--- .../implementation/v3/MongoChecksumsV3.ts | 4 ++-- .../implementation/v3/MongoCompactorV3.ts | 4 ++-- .../v3/MongoParameterLookupV3.ts | 2 +- .../v3/MongoSyncBucketStorageV3.ts | 20 +++++++++---------- .../implementation/v3/PersistedBatchV3.ts | 4 ++-- .../implementation/v3/SourceRecordStoreV3.ts | 6 +++--- .../src/storage/implementation/v3/models.ts | 6 +++--- .../src/storage/storage-index.ts | 14 ++++++------- .../module-mongodb-storage/src/utils/util.ts | 6 +++--- .../test/src/storage_sync.test.ts | 3 +-- 29 files changed, 89 insertions(+), 98 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 46a1697ef..4e90ec55b 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -1,20 +1,20 @@ import { GetIntanceOptions, storage } from '@powersync/service-core'; import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; -import { v4 as uuid } from 'uuid'; import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { v4 as uuid } from 'uuid'; import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; +import { generateSlotName } from '../utils/util.js'; +import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; +import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; +import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; import { PowerSyncMongo } from './implementation/db.js'; import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js'; -import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js'; -import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; -import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js'; -import { generateSlotName } from '../utils/util.js'; import { MongoChecksumOptions } from './implementation/MongoChecksums.js'; -import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; +import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js'; export interface MongoBucketStorageOptions { checksumOptions?: Omit; diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts index 5b6a52db6..fc7bfd672 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -1,10 +1,5 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework'; -import { - BucketDataSource, - ParameterIndexLookupCreator, - ParameterLookupScope, - SyncConfigWithErrors -} from '@powersync/service-sync-rules'; +import { BucketDataSource, ParameterIndexLookupCreator, SyncConfigWithErrors } from '@powersync/service-sync-rules'; import { SyncRuleDocument } from './models.js'; export type BucketDefinitionId = string; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 9807dd92d..770d94a11 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -1,4 +1,3 @@ -import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { HydratedSyncRules, SqlEventDescriptor, SqliteRow, SqliteValue } from '@powersync/service-sync-rules'; import * as bson from 'bson'; @@ -26,15 +25,15 @@ import { } from '@powersync/service-core'; import * as timers from 'node:timers/promises'; import { mongoTableId } from '../../utils/util.js'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { PersistedBatch } from './common/PersistedBatch.js'; +import { LoadedSourceRecord, SourceRecordStore } from './common/SourceRecordStore.js'; import type { VersionedPowerSyncMongo } from './db.js'; import { SyncRuleDocument } from './models.js'; -import { LoadedSourceRecord, SourceRecordStore } from './common/SourceRecordStore.js'; +import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; import { OperationBatch, RecordOperation } from './OperationBatch.js'; -import { PersistedBatch } from './common/PersistedBatch.js'; -import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; // Currently, we can only have a single flush() at a time, since it locks the op_id sequence. // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index ad89bafab..c2b587ecb 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -9,19 +9,19 @@ import { utils } from '@powersync/service-core'; -import type { VersionedPowerSyncMongo } from './db.js'; import { BucketDefinitionId } from './BucketDefinitionMapping.js'; +import type { VersionedPowerSyncMongo } from './db.js'; import { BucketDataDocumentBase, - LEGACY_BUCKET_DATA_DEFINITION_ID, - TaggedBucketDataDocument, + bucketDataDocumentToTagged, BucketStateDocumentBase, - bucketDataDocumentToTagged + LEGACY_BUCKET_DATA_DEFINITION_ID, + TaggedBucketDataDocument } from './models.js'; +import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; +import { cacheKey } from './OperationBatch.js'; import { BucketDataDocumentV1 } from './v1/models.js'; import { BucketDataDocumentV3 } from './v3/models.js'; -import { cacheKey } from './OperationBatch.js'; -import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; interface CurrentBucketState { /** Bucket name */ diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index ac496b55a..94da94c3b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -1,3 +1,5 @@ +import { ServiceAssertionError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; import { BucketDataScope, BucketDataSource, @@ -9,10 +11,8 @@ import { SyncConfigWithErrors, versionedHydrationState } from '@powersync/service-sync-rules'; -import { storage } from '@powersync/service-core'; import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { StorageConfig } from './models.js'; -import { ServiceAssertionError } from '@powersync/lib-services-framework'; export class MongoPersistedSyncRules implements storage.PersistedSyncRules { public readonly hydrationState: HydrationState; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts index e503a229d..f6baf7bab 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts @@ -1,10 +1,10 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js'; import { MongoSyncRulesLock } from './MongoSyncRulesLock.js'; import { PowerSyncMongo } from './db.js'; import { getMongoStorageConfig, SyncRuleDocument } from './models.js'; -import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; -import { MongoPersistedSyncRules } from './MongoPersistedSyncRules.js'; export class MongoPersistedSyncRulesContent extends storage.PersistedSyncRulesContent { public current_lock: MongoSyncRulesLock | null = null; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index cfd45c959..2599d41c0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -25,17 +25,17 @@ import { HydratedSyncRules, ScopedParameterLookup, SqliteJsonRow } from '@powers import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; -import { idPrefixFilter, retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; +import { retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; +import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js'; import type { VersionedPowerSyncMongo } from './db.js'; import { CommonSourceTableDocument, StorageConfig } from './models.js'; +import { MongoBucketBatchOptions } from './MongoBucketBatch.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from './MongoPersistedSyncRulesContent.js'; import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; -import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js'; -import { MongoBucketBatchOptions } from './MongoBucketBatch.js'; export interface MongoSyncBucketStorageOptions { checksumOptions?: Omit; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts index a5965069a..ab058b09a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/MongoSyncBucketStorageContext.ts @@ -1,7 +1,7 @@ import { InternalOpId } from '@powersync/service-core'; +import * as bson from 'bson'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import type { VersionedPowerSyncMongo } from '../db.js'; -import * as bson from 'bson'; export interface MongoSyncBucketStorageContext { db: TDb; diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 46bfbbae6..1f13a4d3b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -4,12 +4,11 @@ import * as bson from 'bson'; import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; +import { mongoTableId } from '../../../utils/util.js'; +import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { MongoIdSequence } from '../MongoIdSequence.js'; import type { VersionedPowerSyncMongo } from '../db.js'; -import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; -import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { TaggedBucketParameterDocument, TaggedBucketDataDocument } from '../models.js'; -import { mongoTableId } from '../../../utils/util.js'; +import { TaggedBucketDataDocument, TaggedBucketParameterDocument } from '../models.js'; import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js'; /** diff --git a/modules/module-mongodb-storage/src/storage/implementation/db.ts b/modules/module-mongodb-storage/src/storage/implementation/db.ts index dc6458ab9..6d4ab93a9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/db.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/db.ts @@ -22,8 +22,8 @@ import { BucketStateDocumentV1, CurrentDataDocument } from './v1/models.js'; -import { BucketDataDocumentV3 } from './v3/models.js'; import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js'; +import { BucketDataDocumentV3 } from './v3/models.js'; import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js'; export interface PowerSyncMongoOptions { diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index f4af4a1e0..9d0029aa7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -1,8 +1,8 @@ +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-core'; import { SqliteJsonValue } from '@powersync/service-sync-rules'; -import * as bson from 'bson'; import { event_types } from '@powersync/service-types'; -import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; +import * as bson from 'bson'; import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js'; import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts index b8b6f70cd..26c39ef12 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts @@ -1,8 +1,8 @@ import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js'; -import { SourceRecordStore } from '../common/SourceRecordStore.js'; -import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; +import { SourceRecordStore } from '../common/SourceRecordStore.js'; import { PersistedBatchV1 } from './PersistedBatchV1.js'; +import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoBucketBatchV1 extends MongoBucketBatch { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts index a8c8a6a38..a363dc430 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts @@ -5,8 +5,7 @@ import { InternalOpId, PartialChecksumMap } from '@powersync/service-core'; -import { FetchPartialBucketChecksumByBucket } from '../MongoChecksums.js'; -import { MongoChecksums } from '../MongoChecksums.js'; +import { FetchPartialBucketChecksumByBucket, MongoChecksums } from '../MongoChecksums.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoChecksumsV1 extends MongoChecksums { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 44dca3f76..351fd564d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -1,17 +1,17 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { BucketDataDocumentBase, BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID, TaggedBucketDataDocument } from '../models.js'; -import { BucketDataKeyV1, BucketStateDocumentV1, taggedBucketDataDocumentToV1 } from './models.js'; -import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; -import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; +import { BucketDataKeyV1, BucketStateDocumentV1, taggedBucketDataDocumentToV1 } from './models.js'; import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoCompactorV1 extends MongoCompactor { // Override types to the more specific ones diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 7e92fc87e..761695175 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -15,12 +15,6 @@ import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-ru import * as bson from 'bson'; import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; -import { MongoChecksums } from '../MongoChecksums.js'; -import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; -import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; -import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; -import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; -import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext @@ -31,6 +25,12 @@ import { LEGACY_BUCKET_DATA_DEFINITION_ID, SourceKey } from '../models.js'; +import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; +import { MongoChecksums } from '../MongoChecksums.js'; +import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; +import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; +import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; import { BucketDataDocumentV1, BucketDataKeyV1, BucketStateDocument } from './models.js'; import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; import { MongoChecksumsV1 } from './MongoChecksumsV1.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index 461f2c29a..a7a8191a6 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -1,17 +1,20 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { JSONBig } from '@powersync/service-jsonbig'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; import * as bson from 'bson'; +import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; import { + BucketStateUpdate, PersistedBatch, SaveBucketDataOptions, SaveParameterDataOptions, UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; import { LEGACY_BUCKET_DATA_DEFINITION_ID, LEGACY_BUCKET_PARAMETER_INDEX_ID, SourceKey } from '../models.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; import { BucketParameterDocument, BucketStateDocumentV1, @@ -19,9 +22,6 @@ import { taggedBucketDataDocumentToV1, taggedBucketParameterDocumentToV1 } from './models.js'; -import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; -import { BucketStateUpdate } from '../common/PersistedBatch.js'; -import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class PersistedBatchV1 extends PersistedBatch { declare protected readonly db: VersionedPowerSyncMongoV1; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts index fcd1af1ef..c5f990159 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SourceRecordStoreV1.ts @@ -1,19 +1,19 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; +import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { idPrefixFilter } from '../../../utils/util.js'; import { cacheKey } from '../OperationBatch.js'; import { + LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordLookupState, - LoadedSourceRecord, SourceRecordStore } from '../common/SourceRecordStore.js'; import { SourceKey } from '../models.js'; -import { CurrentDataDocument } from './models.js'; -import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; +import { CurrentDataDocument } from './models.js'; export class SourceRecordStoreV1 implements SourceRecordStore { constructor( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts index eecdbc559..e6d827321 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts @@ -1,13 +1,13 @@ +import * as bson from 'bson'; import { - BucketParameterDocumentBase, BucketDataDocumentBase, + BucketParameterDocumentBase, BucketStateDocumentBase, SourceKey, SourceTableDocument, TaggedBucketDataDocument, TaggedBucketParameterDocument } from '../models.js'; -import * as bson from 'bson'; export interface BucketDataKeyV1 { /** group_id */ diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts index df74fc399..115548c36 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -1,11 +1,11 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { storage } from '@powersync/service-core'; +import { mongoTableId } from '../../../utils/util.js'; import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js'; -import { SourceRecordStore } from '../common/SourceRecordStore.js'; -import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; +import { SourceRecordStore } from '../common/SourceRecordStore.js'; import { PersistedBatchV3 } from './PersistedBatchV3.js'; -import { mongoTableId } from '../../../utils/util.js'; +import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoBucketBatchV3 extends MongoBucketBatch { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index 590c7348b..7c5bd5af1 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -10,9 +10,9 @@ import { createV3BucketFilter, emptyChecksumForRequest, FetchPartialBucketChecksumV3, - MongoChecksumOptions + MongoChecksumOptions, + MongoChecksums } from '../MongoChecksums.js'; -import { MongoChecksums } from '../MongoChecksums.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoChecksumsV3 extends MongoChecksums { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index c30f969ce..8c6c6378f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -1,9 +1,9 @@ -import { MONGO_OPERATION_TIMEOUT_MS, mongo } from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; import { BucketDataDocumentBase, BucketStateDocumentBase, TaggedBucketDataDocument } from '../models.js'; +import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; import { BucketDataKeyV3, BucketStateDocumentV3, taggedBucketDataDocumentToV3 } from './models.js'; import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts index c4793b7b3..b787608ce 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoParameterLookupV3.ts @@ -1,6 +1,6 @@ -import * as bson from 'bson'; import { deserializeParameterLookup } from '@powersync/service-core'; import { ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; import { ParameterIndexId } from '../BucketDefinitionMapping.js'; export function serializeParameterLookupV3(lookup: ScopedParameterLookup): bson.Binary { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 3978f6c14..14d381402 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -9,29 +9,29 @@ import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { JSONBig } from '@powersync/service-jsonbig'; import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; -import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; -import { bucketDataDocumentToTagged, CommonSourceTableDocument } from '../models.js'; -import { BucketDataDocumentV3, BucketParameterDocumentV3 } from './models.js'; +import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js'; -import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; -import { MongoBucketStorage } from '../../MongoBucketStorage.js'; -import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; +import { bucketDataDocumentToTagged, CommonSourceTableDocument } from '../models.js'; import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; -import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; -import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; -import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; import { MongoChecksums } from '../MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; +import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; +import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; +import { BucketDataDocumentV3, BucketParameterDocumentV3 } from './models.js'; +import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; +import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; import { MongoCompactorV3 } from './MongoCompactorV3.js'; import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js'; +import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { // Declare types to be more specific diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 17049e9ca..187be500b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -4,15 +4,16 @@ import { InternalOpId, storage, utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; import * as bson from 'bson'; import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; -import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { + BucketStateUpdate, PersistedBatch, SaveBucketDataOptions, SaveParameterDataOptions, UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; import { SourceTableKey } from '../models.js'; +import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; import { BucketParameterDocumentV3, BucketStateDocumentV3, @@ -22,7 +23,6 @@ import { taggedBucketParameterDocumentToV3 } from './models.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; -import { BucketStateUpdate } from '../common/PersistedBatch.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class PersistedBatchV3 extends PersistedBatch { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts index db00a2f5a..aae823f9d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/SourceRecordStoreV3.ts @@ -2,15 +2,15 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { Logger } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; -import * as bson from 'bson'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import * as bson from 'bson'; import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js'; +import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { cacheKey } from '../OperationBatch.js'; import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js'; -import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js'; -import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; +import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js'; export class SourceRecordStoreV3 implements SourceRecordStore { constructor( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index 5648540f3..444562c27 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -1,18 +1,18 @@ import { InternalOpId } from '@powersync/service-core'; +import * as bson from 'bson'; import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; import { BucketDataDocumentBase, BucketDataKey, BucketParameterDocumentBase, + BucketStateDocumentBase, CurrentBucket, ReplicaId, SourceTableDocument, SourceTableKey, TaggedBucketDataDocument, - TaggedBucketParameterDocument, - BucketStateDocumentBase + TaggedBucketParameterDocument } from '../models.js'; -import * as bson from 'bson'; export interface CurrentBucketV3 extends CurrentBucket { def: BucketDefinitionId; diff --git a/modules/module-mongodb-storage/src/storage/storage-index.ts b/modules/module-mongodb-storage/src/storage/storage-index.ts index bb0a5de62..bcc83ab1b 100644 --- a/modules/module-mongodb-storage/src/storage/storage-index.ts +++ b/modules/module-mongodb-storage/src/storage/storage-index.ts @@ -1,17 +1,17 @@ -export * from './implementation/db.js'; +export * as test_utils from '../utils/test-utils.js'; +export * from '../utils/util.js'; export * from './implementation/BucketDefinitionMapping.js'; +export * from './implementation/common/PersistedBatch.js'; +export * from './implementation/createMongoSyncBucketStorage.js'; +export * from './implementation/db.js'; export * from './implementation/models.js'; -export * from './implementation/v1/models.js'; -export * from './implementation/v3/models.js'; export * from './implementation/MongoIdSequence.js'; export * from './implementation/MongoPersistedSyncRules.js'; export * from './implementation/MongoPersistedSyncRulesContent.js'; export * from './implementation/MongoStorageProvider.js'; -export * from './implementation/createMongoSyncBucketStorage.js'; export * from './implementation/MongoSyncRulesLock.js'; export * from './implementation/OperationBatch.js'; -export * from './implementation/common/PersistedBatch.js'; -export * from '../utils/util.js'; +export * from './implementation/v1/models.js'; +export * from './implementation/v3/models.js'; export * from './MongoBucketStorage.js'; export * from './MongoReportStorage.js'; -export * as test_utils from '../utils/test-utils.js'; diff --git a/modules/module-mongodb-storage/src/utils/util.ts b/modules/module-mongodb-storage/src/utils/util.ts index 2e0a1cf9c..f8e8b5334 100644 --- a/modules/module-mongodb-storage/src/utils/util.ts +++ b/modules/module-mongodb-storage/src/utils/util.ts @@ -1,11 +1,11 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; +import { mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; import * as bson from 'bson'; import * as crypto from 'crypto'; import * as timers from 'node:timers/promises'; import * as uuid from 'uuid'; -import { mongo } from '@powersync/lib-service-mongodb'; -import { storage, utils } from '@powersync/service-core'; -import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework'; import { TaggedBucketDataDocument } from '../storage/implementation/models.js'; export function idPrefixFilter(prefix: Partial, rest: (keyof T)[]): mongo.Condition { diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index df74fe6c8..6afa1c38b 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -1,6 +1,5 @@ import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; -import { JSONBig } from '@powersync/service-jsonbig'; import { RequestParameters } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; @@ -8,8 +7,8 @@ import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { SyncRuleDocument } from '../../src/storage/implementation/models.js'; import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js'; -import { CurrentBucketV3 } from '../../src/storage/implementation/v3/models.js'; import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; +import { CurrentBucketV3 } from '../../src/storage/implementation/v3/models.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) { From dc8d97a32369cf0866a9e40fd273aeaad659c1d8 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 11:56:40 +0200 Subject: [PATCH 83/93] Cleanup types. --- .../src/storage/implementation/v1/MongoBucketBatchV1.ts | 3 ++- .../src/storage/implementation/v1/models.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts index 26c39ef12..723aa371b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts @@ -1,3 +1,4 @@ +import { SourceTable } from '@powersync/service-core'; import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; import { SourceRecordStore } from '../common/SourceRecordStore.js'; @@ -25,7 +26,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { return this.store; } - protected async cleanupDroppedSourceTables(_sourceTables: import('@powersync/service-core').storage.SourceTable[]) { + protected async cleanupDroppedSourceTables(_tables: SourceTable[]) { // No-op for V1: source records live in a shared collection. } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts index e6d827321..80f29f2d7 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts @@ -3,6 +3,7 @@ import { BucketDataDocumentBase, BucketParameterDocumentBase, BucketStateDocumentBase, + CurrentBucket, SourceKey, SourceTableDocument, TaggedBucketDataDocument, @@ -21,7 +22,7 @@ export interface BucketDataKeyV1 { export interface CurrentDataDocument { _id: SourceKey; data: bson.Binary; - buckets: import('../models.js').CurrentBucket[]; + buckets: CurrentBucket[]; lookups: bson.Binary[]; } From 62cf162750c23114706a6f41cec86bf5b17b33be Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 12:02:47 +0200 Subject: [PATCH 84/93] Reduce type casting in tests. --- .../implementation/MongoSyncBucketStorage.ts | 8 +++++++- .../implementation/v1/MongoSyncBucketStorageV1.ts | 2 +- .../implementation/v3/MongoSyncBucketStorageV3.ts | 2 +- .../test/src/storage_compacting.test.ts | 14 +++----------- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 2599d41c0..a32f76ad0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -87,8 +87,14 @@ export abstract class MongoSyncBucketStorage }); } + /** + * Not for external use - public here for tests only. + * + * @internal + */ + abstract createMongoCompactor(options: MongoCompactOptions): MongoCompactor; + protected abstract createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums; - protected abstract createMongoCompactor(options: MongoCompactOptions): MongoCompactor; protected abstract createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 761695175..69abf0d87 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -68,7 +68,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { }); } - protected createMongoCompactor(options: MongoCompactOptions): MongoCompactor { + createMongoCompactor(options: MongoCompactOptions): MongoCompactor { return new MongoCompactorV1(this, this.db, options); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 14d381402..aaebc6f0d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -84,7 +84,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { }); } - protected createMongoCompactor(options: MongoCompactOptions): MongoCompactor { + createMongoCompactor(options: MongoCompactOptions): MongoCompactor { return new MongoCompactorV3(this, this.db, options); } diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 959e85c52..fdeec583c 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1,8 +1,6 @@ import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; -import { MongoCompactorV1 } from '../../src/storage/implementation/v1/MongoCompactorV1.js'; -import { MongoCompactorV3 } from '../../src/storage/implementation/v3/MongoCompactorV3.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; describe('Mongo Sync Bucket Storage Compact', () => { @@ -213,16 +211,10 @@ bucket_definitions: } await bucketStateCollection.insertOne(bucketStateDocument); - // This test uses a couple of internal APIs of the compactor - there is no simple way - // to test this using the current public APIs. - let compactor: MongoCompactorV1 | MongoCompactorV3; - if (storageDb.storageConfig.incrementalReprocessing) { - compactor = new MongoCompactorV3(bucketStorage as any, storageDb, { maxOpId: 5n }); - } else { - compactor = new MongoCompactorV1(bucketStorage as any, storageDb, { maxOpId: 5n }); - } + // This test uses a couple of "internal" APIs of the compactor. + const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n }); - const dirtyBuckets = (compactor as any).dirtyBucketBatches({ + const dirtyBuckets = compactor.dirtyBucketBatches({ minBucketChanges: 1, minChangeRatio: 0.39 }); From 6a26582ef0483e41b9171aa65b17274ce9510e0e Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 12:05:39 +0200 Subject: [PATCH 85/93] Further improve test types. --- .../test/src/storage_compacting.test.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index fdeec583c..d6605244c 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -1,3 +1,4 @@ +import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; import { storage, SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; @@ -163,16 +164,14 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); - const storageDb = (bucketStorage as any).db; + const storageDb = bucketStorage.db; // This simulates bucket_state created using bigint bytes. // This typically happens when buckets get very large (> 2GiB). We don't want to create that much // data in the tests, so we directly insert the bucket_state here. - let bucketStateCollection; - let bucketStateDocument; if (storageDb.storageConfig.incrementalReprocessing) { - bucketStateCollection = storageDb.bucketStateV3(bucketStorage.group_id); - bucketStateDocument = { + const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id); + await bucketStateCollection.insertOne({ _id: { d: '1', b: 'global[]' @@ -188,10 +187,9 @@ bucket_definitions: count: 2, bytes: 5n } - }; + }); } else { - bucketStateCollection = factory.db.bucket_state; - bucketStateDocument = { + await factory.db.bucket_state.insertOne({ _id: { g: bucketStorage.group_id, b: 'global[]' @@ -207,9 +205,8 @@ bucket_definitions: count: 2, bytes: 5n } - }; + }); } - await bucketStateCollection.insertOne(bucketStateDocument); // This test uses a couple of "internal" APIs of the compactor. const compactor = bucketStorage.createMongoCompactor({ maxOpId: 5n }); From 9a73e170b857c3a7caf40036396d5fab77c94d6b Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 12:08:15 +0200 Subject: [PATCH 86/93] More type cast improvements and comments. --- .../test/src/storage_compacting.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index d6605244c..003cbab59 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -64,11 +64,13 @@ bucket_definitions: test('full compact', async () => { const { bucketStorage, checkpoint, factory, syncRules } = await setup(); - const storageDb = (bucketStorage as any).db; + const storageDb = bucketStorage.db; // Simulate bucket_state from old version not being available if (storageDb.storageConfig.incrementalReprocessing) { - await storageDb.bucketStateV3(bucketStorage.group_id).deleteMany({}); + // This should actually never happen on V3, but we test this anyway. + // Can remove this if it causes issues in the future. + await (storageDb as VersionedPowerSyncMongoV3).bucketStateV3(bucketStorage.group_id).deleteMany({}); } else { await factory.db.bucket_state.deleteMany({}); } From 2f413f3523b3eca10013964a6f60d8198da0dd7f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 12:23:50 +0200 Subject: [PATCH 87/93] Reduce instance state on MongoCompactor. --- .../storage/implementation/MongoCompactor.ts | 339 +++++++++--------- .../implementation/v1/MongoCompactorV1.ts | 8 +- .../implementation/v3/MongoCompactorV3.ts | 8 +- 3 files changed, 171 insertions(+), 184 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index c2b587ecb..a3a173555 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -15,7 +15,6 @@ import { BucketDataDocumentBase, bucketDataDocumentToTagged, BucketStateDocumentBase, - LEGACY_BUCKET_DATA_DEFINITION_ID, TaggedBucketDataDocument } from './models.js'; import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; @@ -57,10 +56,7 @@ interface CurrentBucketState { opBytes: number; } -type CompactBucketDataDocument = Pick< - TaggedBucketDataDocument, - '_id' | 'def' | 'op' | 'table' | 'row_id' | 'source_table' | 'source_key' | 'checksum' | 'target_op' -> & { +type CompactBucketDataDocument = TaggedBucketDataDocument & { size: number | bigint; }; @@ -94,8 +90,6 @@ export interface DirtyBucket { export abstract class MongoCompactor { protected updates: mongo.AnyBulkWriteOperation[] = []; protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; - protected activeBucketDataCollection: mongo.Collection | null = null; - protected activeBucketDefinitionId: BucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; protected readonly idLimitBytes: number; protected readonly moveBatchLimit: number; @@ -336,171 +330,164 @@ export abstract class MongoCompactor { protected async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) { const idLimitBytes = this.idLimitBytes; - const bucketCollection = await this.getBucketDataCollection(bucket, definitionId); - if (bucketCollection == null) { + const bucketContext = await this.getBucketDataContext(bucket, definitionId); + if (bucketContext == null) { return; } - this.activeBucketDataCollection = bucketCollection.collection; - this.activeBucketDefinitionId = bucketCollection.definitionId; - try { - const currentState: CurrentBucketState = { - bucket, - definitionId: bucketCollection.definitionId, - seen: new Map(), - trackingSize: 0, - lastNotPut: null, - opsSincePut: 0, - checksum: 0, - opCount: 0, - opBytes: 0 - }; - - // Constant lower bound. - const lowerBound = this.bucketDataKey(bucket, new mongo.MinKey() as any); - // Upper bound is adjusted for each batch. - let upperBound = this.bucketDataKey(bucket, new mongo.MaxKey() as any); - - while (true) { - this.signal?.throwIfAborted(); + const currentState: CurrentBucketState = { + bucket, + definitionId: bucketContext.definitionId, + seen: new Map(), + trackingSize: 0, + lastNotPut: null, + opsSincePut: 0, + checksum: 0, + opCount: 0, + opBytes: 0 + }; - // Query one batch at a time, to avoid cursor timeouts. - const pipeline = [ - { - $match: { - _id: { - $gte: lowerBound, - $lt: upperBound - }, - // Workaround for a clustered collection bug where the $lt operator may include upperBound. - // Technically only needed for storage V3. - // https://jira.mongodb.org/browse/SERVER-121822 - '_id.o': { $lt: upperBound.o } - } - }, - { $sort: { _id: -1 } }, - { $limit: this.moveBatchQueryLimit }, - { - $project: { - _id: 1, - op: 1, - table: 1, - row_id: 1, - source_table: 1, - source_key: 1, - checksum: 1, - size: { $bsonSize: '$$ROOT' } - } - } - ]; + // Constant lower bound. + const lowerBound = this.bucketDataKey(bucket, new mongo.MinKey() as any); + // Upper bound is adjusted for each batch. + let upperBound = this.bucketDataKey(bucket, new mongo.MaxKey() as any); - const cursor = bucketCollection.collection.aggregate( - pipeline, - { - // batchSize is 1 more than limit to auto-close the cursor. - // See https://github.com/mongodb/node-mongodb-native/pull/4580 - batchSize: this.moveBatchQueryLimit + 1 - } - ); - // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. - // Instead, we load up to the limit. - const rawBatch = await cursor.toArray(); - const batch = rawBatch.map((document) => this.tagBucketDataDocument(document, bucketCollection.definitionId)); + while (true) { + this.signal?.throwIfAborted(); - if (batch.length == 0) { - // We've reached the end. - break; + // Query one batch at a time, to avoid cursor timeouts. + const pipeline = [ + { + $match: { + _id: { + $gte: lowerBound, + $lt: upperBound + }, + // Workaround for a clustered collection bug where the $lt operator may include upperBound. + // Technically only needed for storage V3. + // https://jira.mongodb.org/browse/SERVER-121822 + '_id.o': { $lt: upperBound.o } + } + }, + { $sort: { _id: -1 } }, + { $limit: this.moveBatchQueryLimit }, + { + $project: { + _id: 1, + op: 1, + table: 1, + row_id: 1, + source_table: 1, + source_key: 1, + checksum: 1, + size: { $bsonSize: '$$ROOT' } + } + } + ]; + + const cursor = bucketContext.collection.aggregate( + pipeline, + { + // batchSize is 1 more than limit to auto-close the cursor. + // See https://github.com/mongodb/node-mongodb-native/pull/4580 + batchSize: this.moveBatchQueryLimit + 1 } + ); + // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. + // Instead, we load up to the limit. + const rawBatch = await cursor.toArray(); + const batch = rawBatch.map((document) => this.tagBucketDataDocument(document, bucketContext.definitionId)); - // Reuse the exact collection _id value from Mongo for the next bound. - upperBound = rawBatch[rawBatch.length - 1]._id; + if (batch.length == 0) { + // We've reached the end. + break; + } - for (const doc of batch) { - if (doc._id.o > this.maxOpId) { - continue; - } + // Reuse the exact collection _id value from Mongo for the next bound. + upperBound = rawBatch[rawBatch.length - 1]._id; - currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum)); - currentState.opCount += 1; - - let isPersistentPut = doc.op == 'PUT'; - - currentState.opBytes += Number(doc.size); - if (doc.op == 'REMOVE' || doc.op == 'PUT') { - const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; - const targetOp = currentState.seen.get(key); - if (targetOp) { - // Will convert to MOVE, so don't count as PUT. - isPersistentPut = false; - - this.updates.push({ - updateOne: { - filter: { _id: this.bucketDataKey(doc._id.b, doc._id.o) }, - update: { - $set: { - op: 'MOVE', - target_op: targetOp - }, - $unset: { - source_table: 1, - source_key: 1, - table: 1, - row_id: 1, - data: 1 - } - } satisfies mongo.UpdateFilter - } - }); - - // TODO: better estimate for this. - currentState.opBytes += 200 - Number(doc.size); - } else if (currentState.trackingSize < idLimitBytes) { - // flatstr reduces the memory usage by flattening the string. - currentState.seen.set(utils.flatstr(key), doc._id.o); - // length + 16 for the string - // 24 for the bigint - // 50 for map overhead - // 50 for additional overhead - currentState.trackingSize += key.length + 140; - } - } + for (const doc of batch) { + if (doc._id.o > this.maxOpId) { + continue; + } - if (isPersistentPut) { - currentState.lastNotPut = null; - currentState.opsSincePut = 0; - } else if (doc.op != 'CLEAR') { - if (currentState.lastNotPut == null) { - currentState.lastNotPut = doc._id.o; - } - currentState.opsSincePut += 1; + currentState.checksum = addChecksums(currentState.checksum, Number(doc.checksum)); + currentState.opCount += 1; + + let isPersistentPut = doc.op == 'PUT'; + + currentState.opBytes += Number(doc.size); + if (doc.op == 'REMOVE' || doc.op == 'PUT') { + const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; + const targetOp = currentState.seen.get(key); + if (targetOp) { + // Will convert to MOVE, so don't count as PUT. + isPersistentPut = false; + + this.updates.push({ + updateOne: { + filter: { _id: this.bucketDataKey(doc._id.b, doc._id.o) }, + update: { + $set: { + op: 'MOVE', + target_op: targetOp + }, + $unset: { + source_table: 1, + source_key: 1, + table: 1, + row_id: 1, + data: 1 + } + } satisfies mongo.UpdateFilter + } + }); + + // TODO: better estimate for this. + currentState.opBytes += 200 - Number(doc.size); + } else if (currentState.trackingSize < idLimitBytes) { + // flatstr reduces the memory usage by flattening the string. + currentState.seen.set(utils.flatstr(key), doc._id.o); + // length + 16 for the string + // 24 for the bigint + // 50 for map overhead + // 50 for additional overhead + currentState.trackingSize += key.length + 140; } + } - if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { - await this.flush(); + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (doc.op != 'CLEAR') { + if (currentState.lastNotPut == null) { + currentState.lastNotPut = doc._id.o; } + currentState.opsSincePut += 1; } - logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); + if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { + await this.flush(bucketContext.collection); + } } - // Free memory before clearing the bucket. - currentState.seen.clear(); - if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { - logger.info( - `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` - ); - // Need flush() before clear(). - await this.flush(); - await this.clearBucket(currentState); - } + logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`); + } - // Do this after clearBucket so we have accurate counts. - this.updateBucketChecksums(currentState); - // Need another flush after updateBucketChecksums(). - await this.flush(); - } finally { - this.activeBucketDataCollection = null; - this.activeBucketDefinitionId = LEGACY_BUCKET_DATA_DEFINITION_ID; + // Free memory before clearing the bucket. + currentState.seen.clear(); + if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) { + logger.info( + `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + // Need flush() before clear(). + await this.flush(bucketContext.collection); + await this.clearBucket(currentState, bucketContext); } + + // Do this after clearBucket so we have accurate counts. + this.updateBucketChecksums(currentState); + // Need another flush after updateBucketChecksums(). + await this.flush(bucketContext.collection); } protected updateBucketChecksums(state: CurrentBucketState) { @@ -535,22 +522,24 @@ export abstract class MongoCompactor { }); } - protected async flush() { + protected async flush(collection: mongo.Collection) { if (this.updates.length > 0) { logger.info(`Compacting ${this.updates.length} ops`); - if (this.activeBucketDataCollection == null) { - throw new ServiceAssertionError('No bucket_data collection selected for compaction'); - } - await this.activeBucketDataCollection.bulkWrite(this.updates, { + await collection.bulkWrite(this.updates, { // Order is not important. Since checksums are not affected, these operations can happen in any order, // and it's fine if the operations are partially applied. Each individual operation is atomic. ordered: false }); this.updates = []; } + + await this.flushBucketStateUpdates(); + } + + private async flushBucketStateUpdates() { if (this.bucketStateUpdates.length > 0) { logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`); - await this.flushBucketStateUpdates(); + await this.writeBucketStateUpdates(); this.bucketStateUpdates = []; } } @@ -560,13 +549,9 @@ export abstract class MongoCompactor { * * @param currentState tracks the last non-PUT op, which will be converted to CLEAR. */ - protected async clearBucket(currentState: CurrentBucketState) { + protected async clearBucket(currentState: CurrentBucketState, context: BucketDataCollectionContext) { const bucket = currentState.bucket; const clearOp = currentState.lastNotPut!; - const bucketCollection = this.activeBucketDataCollection; - if (bucketCollection == null) { - throw new ServiceAssertionError('No bucket_data collection selected for compaction'); - } const opFilter = { _id: { @@ -586,7 +571,7 @@ export abstract class MongoCompactor { // We need a transaction per batch to make sure checksums stay consistent. await session.withTransaction( async () => { - const query = bucketCollection.find(opFilter as any, { + const query = context.collection.find(opFilter as any, { session, sort: { _id: 1 }, projection: { @@ -603,10 +588,7 @@ export abstract class MongoCompactor { let gotAnOp = false; let numberOfOpsToClear = 0; for await (const rawOp of query.stream()) { - const op = this.tagClearBucketDataDocument( - rawOp as BucketDataClearProjection, - this.activeBucketDefinitionId - ); + const op = this.tagClearBucketDataDocument(rawOp as BucketDataClearProjection, context.definitionId); if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') { checksum = utils.addChecksums(checksum, Number(op.checksum)); @@ -630,7 +612,7 @@ export abstract class MongoCompactor { } logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?._id.o}`); - await bucketCollection.deleteMany( + await context.collection.deleteMany( { _id: { $gte: this.bucketDataKey(bucket, new mongo.MinKey()), @@ -640,9 +622,9 @@ export abstract class MongoCompactor { { session } ); - await bucketCollection.insertOne( + await context.collection.insertOne( this.collectionBucketDataDocument({ - def: this.activeBucketDefinitionId, + def: context.definitionId, _id: lastOp!._id, op: 'CLEAR', checksum: BigInt(checksum), @@ -704,7 +686,7 @@ export abstract class MongoCompactor { }); } - await this.flush(); + await this.flushBucketStateUpdates(); } protected tagBucketDataDocument( @@ -740,7 +722,7 @@ export abstract class MongoCompactor { return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; } - protected abstract flushBucketStateUpdates(): Promise; + protected abstract writeBucketStateUpdates(): Promise; protected abstract computeChecksumsForBuckets( buckets: Pick[] ): Promise; @@ -749,9 +731,14 @@ export abstract class MongoCompactor { bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey ): BucketDataDocumentBase['_id']; - protected abstract getBucketDataCollection( + protected abstract getBucketDataContext( bucket: string, definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null>; + ): Promise; protected abstract collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase; } + +export interface BucketDataCollectionContext { + definitionId: BucketDefinitionId; + collection: mongo.Collection; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 351fd564d..8b662b6f3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -8,7 +8,7 @@ import { LEGACY_BUCKET_DATA_DEFINITION_ID, TaggedBucketDataDocument } from '../models.js'; -import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { BucketDataCollectionContext, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; import { BucketDataKeyV1, BucketStateDocumentV1, taggedBucketDataDocumentToV1 } from './models.js'; import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; @@ -53,7 +53,7 @@ export class MongoCompactorV1 extends MongoCompactor { ); } - protected async flushBucketStateUpdates(): Promise { + protected async writeBucketStateUpdates(): Promise { await this.db.bucketStateV1.bulkWrite( this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { ordered: false } @@ -91,10 +91,10 @@ export class MongoCompactorV1 extends MongoCompactor { }; } - protected async getBucketDataCollection( + protected async getBucketDataContext( _bucket: string, _definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { + ): Promise { return { collection: this.db.v1_bucket_data as unknown as mongo.Collection, definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index 8c6c6378f..d96d01108 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -3,7 +3,7 @@ import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib import { InternalOpId, storage } from '@powersync/service-core'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { BucketDataDocumentBase, BucketStateDocumentBase, TaggedBucketDataDocument } from '../models.js'; -import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { BucketDataCollectionContext, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; import { BucketDataKeyV3, BucketStateDocumentV3, taggedBucketDataDocumentToV3 } from './models.js'; import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; @@ -42,7 +42,7 @@ export class MongoCompactorV3 extends MongoCompactor { ); } - protected async flushBucketStateUpdates(): Promise { + protected async writeBucketStateUpdates(): Promise { await this.db .bucketStateV3(this.group_id) .bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation[], { ordered: false }); @@ -84,10 +84,10 @@ export class MongoCompactorV3 extends MongoCompactor { return { b: bucket, o: opId as any }; } - protected async getBucketDataCollection( + protected async getBucketDataContext( bucket: string, definitionId: BucketDefinitionId | null - ): Promise<{ collection: mongo.Collection; definitionId: BucketDefinitionId } | null> { + ): Promise { if (definitionId == null) { // Not the _most_ efficient approach, but this is not used often const allDefinitionIds = this.storage.mapping.allBucketDefinitionIds(); From 39c40903d6b313861024b16ea1d29c68ac0fc1ad Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 16:43:11 +0200 Subject: [PATCH 88/93] Introduce SingleBucketStore. --- .../storage/implementation/MongoCompactor.ts | 182 +++++++----------- .../implementation/common/BucketDataDoc.ts | 37 ++++ .../implementation/common/PersistedBatch.ts | 24 +-- .../common/SingleBucketStore.ts | 63 ++++++ .../src/storage/implementation/models.ts | 29 +-- .../implementation/v1/MongoChecksumsV1.ts | 2 +- .../implementation/v1/MongoCompactorV1.ts | 40 ++-- .../v1/MongoSyncBucketStorageV1.ts | 16 +- .../implementation/v1/PersistedBatchV1.ts | 17 +- .../implementation/v1/SingleBucketStoreV1.ts | 83 ++++++++ .../v1/VersionedPowerSyncMongoV1.ts | 2 +- .../src/storage/implementation/v1/models.ts | 29 ++- .../implementation/v3/MongoCompactorV3.ts | 28 +-- .../v3/MongoSyncBucketStorageV3.ts | 12 +- .../implementation/v3/PersistedBatchV3.ts | 17 +- .../implementation/v3/SingleBucketStoreV3.ts | 82 ++++++++ .../src/storage/implementation/v3/models.ts | 29 ++- .../module-mongodb-storage/src/utils/util.ts | 8 +- .../test/src/slow_tests.test.ts | 95 ++------- .../test/src/wal_stream_utils.ts | 41 +--- .../src/test-utils/StorageDataHelpers.ts | 44 +++++ .../src/test-utils/test-utils-index.ts | 1 + 22 files changed, 527 insertions(+), 354 deletions(-) create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/BucketDataDoc.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/common/SingleBucketStore.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts create mode 100644 modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts create mode 100644 packages/service-core-tests/src/test-utils/StorageDataHelpers.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts index a3a173555..78f6fb384 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoCompactor.ts @@ -10,17 +10,12 @@ import { } from '@powersync/service-core'; import { BucketDefinitionId } from './BucketDefinitionMapping.js'; +import { BucketDataDoc, BucketKey } from './common/BucketDataDoc.js'; +import { BucketDataDocumentGeneric, SingleBucketStore } from './common/SingleBucketStore.js'; import type { VersionedPowerSyncMongo } from './db.js'; -import { - BucketDataDocumentBase, - bucketDataDocumentToTagged, - BucketStateDocumentBase, - TaggedBucketDataDocument -} from './models.js'; +import { BucketStateDocumentBase } from './models.js'; import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js'; import { cacheKey } from './OperationBatch.js'; -import { BucketDataDocumentV1 } from './v1/models.js'; -import { BucketDataDocumentV3 } from './v3/models.js'; interface CurrentBucketState { /** Bucket name */ @@ -56,18 +51,7 @@ interface CurrentBucketState { opBytes: number; } -type CompactBucketDataDocument = TaggedBucketDataDocument & { - size: number | bigint; -}; - -type CompactClearBucketDataDocument = Pick; -type BucketDataCollectionDocument = BucketDataDocumentV1 | BucketDataDocumentV3; -type BucketDataClearProjection = { - _id: BucketDataDocumentBase['_id']; - op: CompactClearBucketDataDocument['op']; - checksum: bigint; - target_op?: bigint | null; -}; +type CompactClearProperties = 'op' | 'checksum' | 'target_op'; export interface MongoCompactOptions extends storage.CompactOptions {} @@ -88,7 +72,7 @@ export interface DirtyBucket { } export abstract class MongoCompactor { - protected updates: mongo.AnyBulkWriteOperation[] = []; + protected updates: mongo.AnyBulkWriteOperation[] = []; protected bucketStateUpdates: mongo.AnyBulkWriteOperation[] = []; protected readonly idLimitBytes: number; @@ -336,7 +320,7 @@ export abstract class MongoCompactor { } const currentState: CurrentBucketState = { bucket, - definitionId: bucketContext.definitionId, + definitionId: bucketContext.key.definitionId, seen: new Map(), trackingSize: 0, lastNotPut: null, @@ -347,9 +331,9 @@ export abstract class MongoCompactor { }; // Constant lower bound. - const lowerBound = this.bucketDataKey(bucket, new mongo.MinKey() as any); + const lowerBound = bucketContext.minId; // Upper bound is adjusted for each batch. - let upperBound = this.bucketDataKey(bucket, new mongo.MaxKey() as any); + let upperBound = bucketContext.maxId; while (true) { this.signal?.throwIfAborted(); @@ -384,7 +368,7 @@ export abstract class MongoCompactor { } ]; - const cursor = bucketContext.collection.aggregate( + const cursor = bucketContext.collection.aggregate( pipeline, { // batchSize is 1 more than limit to auto-close the cursor. @@ -395,7 +379,13 @@ export abstract class MongoCompactor { // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns. // Instead, we load up to the limit. const rawBatch = await cursor.toArray(); - const batch = rawBatch.map((document) => this.tagBucketDataDocument(document, bucketContext.definitionId)); + const batch = rawBatch.map((document) => { + const { size, ...rest } = document; + return { + doc: bucketContext.fromPersistedDocument(rest), + size + }; + }); if (batch.length == 0) { // We've reached the end. @@ -405,8 +395,8 @@ export abstract class MongoCompactor { // Reuse the exact collection _id value from Mongo for the next bound. upperBound = rawBatch[rawBatch.length - 1]._id; - for (const doc of batch) { - if (doc._id.o > this.maxOpId) { + for (const { doc, size } of batch) { + if (doc.o > this.maxOpId) { continue; } @@ -415,7 +405,7 @@ export abstract class MongoCompactor { let isPersistentPut = doc.op == 'PUT'; - currentState.opBytes += Number(doc.size); + currentState.opBytes += Number(size); if (doc.op == 'REMOVE' || doc.op == 'PUT') { const key = `${doc.table}/${doc.row_id}/${cacheKey(doc.source_table!, doc.source_key!)}`; const targetOp = currentState.seen.get(key); @@ -425,7 +415,7 @@ export abstract class MongoCompactor { this.updates.push({ updateOne: { - filter: { _id: this.bucketDataKey(doc._id.b, doc._id.o) }, + filter: { _id: bucketContext.docId(doc.o) }, update: { $set: { op: 'MOVE', @@ -438,15 +428,15 @@ export abstract class MongoCompactor { row_id: 1, data: 1 } - } satisfies mongo.UpdateFilter + } satisfies mongo.UpdateFilter } }); // TODO: better estimate for this. - currentState.opBytes += 200 - Number(doc.size); + currentState.opBytes += 200 - Number(size); } else if (currentState.trackingSize < idLimitBytes) { // flatstr reduces the memory usage by flattening the string. - currentState.seen.set(utils.flatstr(key), doc._id.o); + currentState.seen.set(utils.flatstr(key), doc.o); // length + 16 for the string // 24 for the bigint // 50 for map overhead @@ -460,13 +450,13 @@ export abstract class MongoCompactor { currentState.opsSincePut = 0; } else if (doc.op != 'CLEAR') { if (currentState.lastNotPut == null) { - currentState.lastNotPut = doc._id.o; + currentState.lastNotPut = doc.o; } currentState.opsSincePut += 1; } if (this.updates.length + this.bucketStateUpdates.length >= this.moveBatchLimit) { - await this.flush(bucketContext.collection); + await this.flush(bucketContext); } } @@ -480,14 +470,14 @@ export abstract class MongoCompactor { `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` ); // Need flush() before clear(). - await this.flush(bucketContext.collection); + await this.flush(bucketContext); await this.clearBucket(currentState, bucketContext); } // Do this after clearBucket so we have accurate counts. this.updateBucketChecksums(currentState); // Need another flush after updateBucketChecksums(). - await this.flush(bucketContext.collection); + await this.flush(bucketContext); } protected updateBucketChecksums(state: CurrentBucketState) { @@ -522,10 +512,10 @@ export abstract class MongoCompactor { }); } - protected async flush(collection: mongo.Collection) { + protected async flush(col: SingleBucketStore) { if (this.updates.length > 0) { logger.info(`Compacting ${this.updates.length} ops`); - await collection.bulkWrite(this.updates, { + await col.collection.bulkWrite(this.updates, { // Order is not important. Since checksums are not affected, these operations can happen in any order, // and it's fine if the operations are partially applied. Each individual operation is atomic. ordered: false @@ -549,14 +539,13 @@ export abstract class MongoCompactor { * * @param currentState tracks the last non-PUT op, which will be converted to CLEAR. */ - protected async clearBucket(currentState: CurrentBucketState, context: BucketDataCollectionContext) { - const bucket = currentState.bucket; + protected async clearBucket(currentState: CurrentBucketState, col: SingleBucketStore) { const clearOp = currentState.lastNotPut!; const opFilter = { _id: { - $gte: this.bucketDataKey(bucket, new mongo.MinKey() as any), - $lte: this.bucketDataKey(bucket, clearOp) + $gte: col.minId, + $lte: col.docId(clearOp) } }; @@ -571,24 +560,27 @@ export abstract class MongoCompactor { // We need a transaction per batch to make sure checksums stay consistent. await session.withTransaction( async () => { - const query = context.collection.find(opFilter as any, { - session, - sort: { _id: 1 }, - projection: { - _id: 1, - op: 1, - checksum: 1, - target_op: 1 - }, - limit: this.clearBatchLimit - }); + const query = col.collection.find>( + opFilter, + { + session, + sort: { _id: 1 }, + projection: { + _id: 1, + op: 1, + checksum: 1, + target_op: 1 + }, + limit: this.clearBatchLimit + } + ); let checksum = 0; - let lastOp: CompactClearBucketDataDocument | null = null; + let lastOp: Pick | null = null; let targetOp: bigint | null = null; let gotAnOp = false; let numberOfOpsToClear = 0; for await (const rawOp of query.stream()) { - const op = this.tagClearBucketDataDocument(rawOp as BucketDataClearProjection, context.definitionId); + const op = col.fromPartialPersistedDocument(rawOp); if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') { checksum = utils.addChecksums(checksum, Number(op.checksum)); @@ -601,9 +593,7 @@ export abstract class MongoCompactor { targetOp = op.target_op; } } else { - throw new ReplicationAssertionError( - `Unexpected ${op.op} operation at ${this.formatBucketDataKey(op._id)}` - ); + throw new ReplicationAssertionError(`Unexpected ${op.op} operation at ${this.formatBucketDataKey(op)}`); } } if (!gotAnOp) { @@ -611,28 +601,25 @@ export abstract class MongoCompactor { return; } - logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?._id.o}`); - await context.collection.deleteMany( + logger.info(`Flushing CLEAR for ${numberOfOpsToClear} ops at ${lastOp?.o}`); + await col.collection.deleteMany( { _id: { - $gte: this.bucketDataKey(bucket, new mongo.MinKey()), - $lte: this.bucketDataKey(lastOp!._id.b, lastOp!._id.o) + $gte: col.minId, + $lte: col.docId(lastOp!.o) } }, { session } ); - await context.collection.insertOne( - this.collectionBucketDataDocument({ - def: context.definitionId, - _id: lastOp!._id, - op: 'CLEAR', - checksum: BigInt(checksum), - data: null, - target_op: targetOp - }), - { session } - ); + const op = col.toPersistedDocument({ + o: lastOp!.o, + op: 'CLEAR', + checksum: BigInt(checksum), + data: null, + target_op: targetOp + }); + await col.collection.insertOne(op, { session }); opCountDiff = -numberOfOpsToClear + 1; }, @@ -689,37 +676,8 @@ export abstract class MongoCompactor { await this.flushBucketStateUpdates(); } - protected tagBucketDataDocument( - document: BucketDataCollectionDocument & { size: number | bigint }, - definitionId: BucketDefinitionId - ): CompactBucketDataDocument { - const tagged = bucketDataDocumentToTagged(document, definitionId); - return { - ...tagged, - size: document.size - }; - } - - protected tagClearBucketDataDocument( - document: BucketDataClearProjection, - definitionId: BucketDefinitionId - ): CompactClearBucketDataDocument { - return { - def: definitionId, - _id: { - b: document._id.b, - o: document._id.o - }, - op: document.op, - checksum: document.checksum, - target_op: document.target_op - }; - } - - protected formatBucketDataKey(key: BucketDataDocumentBase['_id'] | { _id: BucketDataDocumentBase['_id'] }) { - const bucket = 'b' in key ? key.b : key._id.b; - const op = 'o' in key ? key.o : key._id.o; - return `${this.group_id}:${bucket ?? '?'}:${op ?? '?'}`; + protected formatBucketDataKey(doc: Pick) { + return `${doc.bucketKey.replicationStreamId}:${doc.bucketKey.bucket}:${doc.o}`; } protected abstract writeBucketStateUpdates(): Promise; @@ -727,18 +685,14 @@ export abstract class MongoCompactor { buckets: Pick[] ): Promise; protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document; - protected abstract bucketDataKey( - bucket: string, - opId: InternalOpId | mongo.MinKey | mongo.MaxKey - ): BucketDataDocumentBase['_id']; + protected abstract getBucketDataContext( bucket: string, definitionId: BucketDefinitionId | null - ): Promise; - protected abstract collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase; + ): Promise; } -export interface BucketDataCollectionContext { - definitionId: BucketDefinitionId; - collection: mongo.Collection; +export interface BucketDataCollectionContext { + bucketKey: BucketKey; + collection: mongo.Collection; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/BucketDataDoc.ts b/modules/module-mongodb-storage/src/storage/implementation/common/BucketDataDoc.ts new file mode 100644 index 000000000..93e54f6d9 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/BucketDataDoc.ts @@ -0,0 +1,37 @@ +import { InternalOpId } from '@powersync/service-core'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { BucketDataProperties } from '../models.js'; + +/** + * Full context identifying a bucket. + */ +export interface BucketKey { + /** + * Also referred to as g / group_id. + */ + replicationStreamId: number; + /** + * Bucket definition id, '0' for storage V1. + */ + definitionId: BucketDefinitionId; + /** + * Bucket name. + */ + bucket: string; +} + +/** + * In-memory bucket data document. + * + * This is converted to/from BucketDataDocumentV1 / BucketDataDocumentV3 for storage. + */ +export interface BucketDataDoc extends BucketDataProperties { + /** + * Identifies the bucket for this document. + */ + bucketKey: BucketKey; + /** + * op_id + */ + o: InternalOpId; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 1f13a4d3b..485d3788f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -8,7 +8,8 @@ import { mongoTableId } from '../../../utils/util.js'; import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { MongoIdSequence } from '../MongoIdSequence.js'; import type { VersionedPowerSyncMongo } from '../db.js'; -import { TaggedBucketDataDocument, TaggedBucketParameterDocument } from '../models.js'; +import { TaggedBucketParameterDocument } from '../models.js'; +import { BucketDataDoc, BucketKey } from './BucketDataDoc.js'; import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js'; /** @@ -68,7 +69,7 @@ export interface PersistedBatchOptions { */ export abstract class PersistedBatch { logger: Logger; - bucketData: TaggedBucketDataDocument[] = []; + bucketData: BucketDataDoc[] = []; bucketParameters: TaggedBucketParameterDocument[] = []; bucketStates: Map = new Map(); @@ -148,7 +149,7 @@ export abstract class PersistedBatch { protected addBucketDataPut(options: { op_id: InternalOpId; - definitionId: BucketDefinitionId; + bucketKey: BucketKey; bucket: string; sourceTableId: storage.SourceTable['id']; sourceKey: storage.ReplicaId; @@ -158,11 +159,8 @@ export abstract class PersistedBatch { data: string; }) { this.bucketData.push({ - def: options.definitionId, - _id: { - b: options.bucket, - o: options.op_id - }, + bucketKey: options.bucketKey, + o: options.op_id, op: 'PUT', source_table: mongoTableId(options.sourceTableId), source_key: options.sourceKey, @@ -175,8 +173,7 @@ export abstract class PersistedBatch { protected addBucketDataRemove(options: { op_id: InternalOpId; - definitionId: BucketDefinitionId; - bucket: string; + bucketKey: BucketKey; sourceTableId: storage.SourceTable['id']; sourceKey: storage.ReplicaId; table: string; @@ -184,11 +181,8 @@ export abstract class PersistedBatch { checksum: bigint; }) { this.bucketData.push({ - def: options.definitionId, - _id: { - b: options.bucket, - o: options.op_id - }, + bucketKey: options.bucketKey, + o: options.op_id, op: 'REMOVE', source_table: mongoTableId(options.sourceTableId), source_key: options.sourceKey, diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/SingleBucketStore.ts b/modules/module-mongodb-storage/src/storage/implementation/common/SingleBucketStore.ts new file mode 100644 index 000000000..af04ad23c --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/common/SingleBucketStore.ts @@ -0,0 +1,63 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { InternalOpId } from '@powersync/service-core'; +import { BucketDataProperties } from '../models.js'; +import { BucketDataDoc, BucketKey } from './BucketDataDoc.js'; + +const GENERIC_ID = Symbol('BucketDataDocumentGenericId'); +export type BucketDataDocumentGenericId = { + b: string; + o: InternalOpId; + // Hack to ensure this can't be constructed directly + [GENERIC_ID]: true; +}; + +/** + * This document is never actually constructed - we use it as a "virtual" type. + * + * The actual implementations are BucketDataDocumentV1 or BucketDataDocumentV3. + * They don't fully satisfy this interface, but this works to share common implementations. + * + * The idea is that we can have a common implementation between V1 & V3, using this type, + * and operate on MongoDB collections. + * + * This interface serves two primary purposes: + * 1. Captures properties that exist on both V1 and V3 storage models. + * 2. Gives a common reference when querying or modifying collections. + * + * Generics would've been ideal, but they don't play well with MongoDB collections. + */ +export interface BucketDataDocumentGeneric extends BucketDataProperties { + _id: BucketDataDocumentGenericId; +} + +/** + * Represent read/write access for a single bucket. + * + * This does not implement the actual collection operations, but supports the required conversions + * between in-memory BucketDataDoc and the specific storage formats. + */ +export interface SingleBucketStore { + readonly key: BucketKey; + + readonly collection: mongo.Collection; + docId(o: InternalOpId): BucketDataDocumentGenericId; + readonly minId: BucketDataDocumentGenericId; + readonly maxId: BucketDataDocumentGenericId; + + /** + * Convert in-memory document -> persisted document. + */ + toPersistedDocument(source: Omit): BucketDataDocumentGeneric; + + /** + * Convert persisted document -> in-memory document. + */ + fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc; + + /** + * Convert partial persisted document -> partial in-memory document. + */ + fromPartialPersistedDocument( + doc: Pick + ): Pick; +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/models.ts b/modules/module-mongodb-storage/src/storage/implementation/models.ts index 9d0029aa7..6c09fc692 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/models.ts @@ -3,7 +3,7 @@ import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-co import { SqliteJsonValue } from '@powersync/service-sync-rules'; import { event_types } from '@powersync/service-types'; import * as bson from 'bson'; -import { BucketDefinitionId, ParameterIndexId } from './BucketDefinitionMapping.js'; +import { ParameterIndexId } from './BucketDefinitionMapping.js'; import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js'; import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js'; @@ -68,6 +68,9 @@ export function bucketParameterDocumentToTagged( - document: TDocument, - definitionId: BucketDefinitionId -): TaggedBucketDataDocument { - return { - ...document, - def: definitionId, - _id: { - b: document._id.b, - o: document._id.o - } - }; -} - export interface SourceTableDocument { _id: bson.ObjectId; connection_id: number; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts index a363dc430..62636d37c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoChecksumsV1.ts @@ -14,7 +14,7 @@ export class MongoChecksumsV1 extends MongoChecksums { async computePartialChecksumsDirectByBucket( batch: FetchPartialBucketChecksumByBucket[] ): Promise { - return this.computePartialChecksumsForCollection(batch, this.db.v1_bucket_data, (request) => ({ + return this.computePartialChecksumsForCollection(batch, this.db.bucketDataV1, (request) => ({ _id: { $gt: { g: this.group_id, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts index 8b662b6f3..b19e94c53 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoCompactorV1.ts @@ -1,16 +1,13 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; -import { InternalOpId, storage } from '@powersync/service-core'; +import { storage } from '@powersync/service-core'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { - BucketDataDocumentBase, - BucketStateDocumentBase, - LEGACY_BUCKET_DATA_DEFINITION_ID, - TaggedBucketDataDocument -} from '../models.js'; -import { BucketDataCollectionContext, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; -import { BucketDataKeyV1, BucketStateDocumentV1, taggedBucketDataDocumentToV1 } from './models.js'; +import { SingleBucketStore } from '../common/SingleBucketStore.js'; +import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js'; +import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { BucketStateDocumentV1 } from './models.js'; import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js'; +import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; export class MongoCompactorV1 extends MongoCompactor { @@ -83,25 +80,14 @@ export class MongoCompactorV1 extends MongoCompactor { }; } - protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): BucketDataKeyV1 { - return { - g: this.group_id, - b: bucket, - o: opId as any - }; - } - protected async getBucketDataContext( - _bucket: string, + bucket: string, _definitionId: BucketDefinitionId | null - ): Promise { - return { - collection: this.db.v1_bucket_data as unknown as mongo.Collection, - definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID - }; - } - - protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase { - return taggedBucketDataDocumentToV1(this.group_id, document); + ): Promise { + return new SingleBucketStoreV1(this.db, { + replicationStreamId: this.group_id, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, + bucket + }); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 69abf0d87..77fae1869 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -19,19 +19,14 @@ import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js'; -import { - bucketDataDocumentToTagged, - CommonSourceTableDocument, - LEGACY_BUCKET_DATA_DEFINITION_ID, - SourceKey -} from '../models.js'; +import { CommonSourceTableDocument, SourceKey } from '../models.js'; import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; import { MongoChecksums } from '../MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; -import { BucketDataDocumentV1, BucketDataKeyV1, BucketStateDocument } from './models.js'; +import { BucketDataDocumentV1, BucketDataKeyV1, BucketStateDocument, loadBucketDataDocumentV1 } from './models.js'; import { MongoBucketBatchV1 } from './MongoBucketBatchV1.js'; import { MongoChecksumsV1 } from './MongoChecksumsV1.js'; import { MongoCompactorV1 } from './MongoCompactorV1.js'; @@ -310,11 +305,10 @@ export async function* getBucketDataBatchV1( let targetOp: InternalOpId | null = null; for (let rawData of data) { - const row = bucketDataDocumentToTagged( - bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV1, - LEGACY_BUCKET_DATA_DEFINITION_ID + const row = loadBucketDataDocumentV1( + bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV1 ); - const bucket = row._id.b; + const bucket = row.bucketKey.bucket; if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { let start: ProtocolOpId | undefined = undefined; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index a7a8191a6..3a2f571fa 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -62,8 +62,12 @@ export class PersistedBatchV1 extends PersistedBatch { this.debugLastOpId = op_id; this.addBucketDataPut({ + bucketKey: { + replicationStreamId: this.group_id, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, + bucket: evaluated.bucket + }, op_id, - definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, bucket: evaluated.bucket, sourceTableId: options.table.id, sourceKey: options.sourceKey, @@ -80,9 +84,12 @@ export class PersistedBatchV1 extends PersistedBatch { this.debugLastOpId = op_id; this.addBucketDataRemove({ + bucketKey: { + replicationStreamId: this.group_id, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, + bucket: bucket.bucket + }, op_id, - definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, - bucket: bucket.bucket, sourceTableId: options.table.id, sourceKey: options.sourceKey, table: bucket.table, @@ -203,10 +210,10 @@ export class PersistedBatchV1 extends PersistedBatch { } protected async flushBucketData(session: mongo.ClientSession) { - await this.db.v1_bucket_data.bulkWrite( + await this.db.bucketDataV1.bulkWrite( this.bucketData.map((document) => ({ insertOne: { - document: taggedBucketDataDocumentToV1(this.group_id, document) + document: taggedBucketDataDocumentToV1(document) } })), { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts new file mode 100644 index 000000000..dac3d7f61 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts @@ -0,0 +1,83 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { InternalOpId } from '@powersync/service-core'; +import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js'; +import { + BucketDataDocumentGeneric, + BucketDataDocumentGenericId, + SingleBucketStore +} from '../common/SingleBucketStore.js'; +import { BucketDataProperties } from '../models.js'; +import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; +import { BucketDataDocumentV1, BucketDataKeyV1 } from './models.js'; + +export class SingleBucketStoreV1 implements SingleBucketStore { + public readonly collection: mongo.Collection; + + constructor( + private db: VersionedPowerSyncMongoV1, + public readonly key: BucketKey + ) { + this.collection = db.bucketDataV1 as unknown as mongo.Collection; + } + + docId(o: InternalOpId): BucketDataDocumentGenericId { + // `satisfies BucketDataKeyV1` checks that we use the correct type for V1 storage + // `as anyt` is to allow casting to the interface virtual type + return { + g: this.key.replicationStreamId, + b: this.key.bucket, + o + } satisfies BucketDataKeyV1 as any; + } + + get minId(): BucketDataDocumentGenericId { + return { + g: this.key.replicationStreamId, + b: this.key.bucket, + o: new mongo.MinKey() + } as any; + } + + get maxId(): BucketDataDocumentGenericId { + return { + g: this.key.replicationStreamId, + b: this.key.bucket, + o: new mongo.MaxKey() + } as any; + } + + toPersistedDocument(source: Omit): BucketDataDocumentGeneric { + const { o, ...rest } = source; + const doc: BucketDataDocumentV1 = { + _id: { + g: this.key.replicationStreamId, + b: this.key.bucket, + o: o + }, + ...rest + }; + return doc as unknown as BucketDataDocumentGeneric; + } + + fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc { + const document = doc as unknown as BucketDataDocumentV1; + const { _id, ...rest } = document; + return { + bucketKey: this.key, + o: _id.o, + ...rest + }; + } + + fromPartialPersistedDocument( + doc: Pick + ): Pick { + const document = doc as Pick; + const { _id, ...rest } = document; + return { + bucketKey: this.key, + o: _id.o, + ...rest + } as Pick; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts index 15a99f85e..0758dd4f2 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts @@ -18,7 +18,7 @@ export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo { async initializeStreamStorage(_replicationStreamId: number): Promise {} - get v1_bucket_data(): mongo.Collection { + get bucketDataV1(): mongo.Collection { return this.upstream.bucket_data; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts index 80f29f2d7..73711fd26 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts @@ -1,12 +1,13 @@ import * as bson from 'bson'; +import { BucketDataDoc } from '../common/BucketDataDoc.js'; import { BucketDataDocumentBase, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, + LEGACY_BUCKET_DATA_DEFINITION_ID, SourceKey, SourceTableDocument, - TaggedBucketDataDocument, TaggedBucketParameterDocument } from '../models.js'; @@ -32,21 +33,31 @@ export interface BucketDataDocumentV1 extends BucketDataDocumentBase { _id: BucketDataKeyV1; } -export function taggedBucketDataDocumentToV1( - groupId: number, - document: TaggedBucketDataDocument -): BucketDataDocumentV1 { - const { def: _definitionId, _id: _id, ...rest } = document; +export function taggedBucketDataDocumentToV1(document: BucketDataDoc): BucketDataDocumentV1 { + const { bucketKey, o, ...rest } = document; return { _id: { - g: groupId, - b: _id.b, - o: _id.o + g: bucketKey.replicationStreamId, + b: bucketKey.bucket, + o: o }, ...rest }; } +export function loadBucketDataDocumentV1(doc: BucketDataDocumentV1): BucketDataDoc { + const { _id, ...rest } = doc; + return { + bucketKey: { + replicationStreamId: _id.g, + definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, + bucket: _id.b + }, + o: _id.o, + ...rest + }; +} + export function taggedBucketParameterDocumentToV1(document: TaggedBucketParameterDocument): BucketParameterDocument { const { index: _index, ...rest } = document; return rest as BucketParameterDocument; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts index d96d01108..b9641ca8d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoCompactorV3.ts @@ -1,11 +1,13 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework'; -import { InternalOpId, storage } from '@powersync/service-core'; +import { storage } from '@powersync/service-core'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; -import { BucketDataDocumentBase, BucketStateDocumentBase, TaggedBucketDataDocument } from '../models.js'; -import { BucketDataCollectionContext, DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; -import { BucketDataKeyV3, BucketStateDocumentV3, taggedBucketDataDocumentToV3 } from './models.js'; +import { SingleBucketStore } from '../common/SingleBucketStore.js'; +import { BucketStateDocumentBase } from '../models.js'; +import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js'; +import { BucketStateDocumentV3 } from './models.js'; import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js'; +import { SingleBucketStoreV3 } from './SingleBucketStoreV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoCompactorV3 extends MongoCompactor { @@ -80,14 +82,10 @@ export class MongoCompactorV3 extends MongoCompactor { }; } - protected bucketDataKey(bucket: string, opId: InternalOpId | mongo.MinKey | mongo.MaxKey): BucketDataKeyV3 { - return { b: bucket, o: opId as any }; - } - protected async getBucketDataContext( bucket: string, definitionId: BucketDefinitionId | null - ): Promise { + ): Promise { if (definitionId == null) { // Not the _most_ efficient approach, but this is not used often const allDefinitionIds = this.storage.mapping.allBucketDefinitionIds(); @@ -104,16 +102,6 @@ export class MongoCompactorV3 extends MongoCompactor { definitionId = bucketState._id.d; } - return { - collection: this.db.bucketDataV3( - this.group_id, - definitionId - ) as unknown as mongo.Collection, - definitionId - }; - } - - protected collectionBucketDataDocument(document: TaggedBucketDataDocument): BucketDataDocumentBase { - return taggedBucketDataDocumentToV3(document); + return new SingleBucketStoreV3(this.db, { bucket, definitionId, replicationStreamId: this.group_id }); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index aaebc6f0d..a280a148b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -18,14 +18,14 @@ import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js'; -import { bucketDataDocumentToTagged, CommonSourceTableDocument } from '../models.js'; +import { CommonSourceTableDocument } from '../models.js'; import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; import { MongoChecksums } from '../MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js'; import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; -import { BucketDataDocumentV3, BucketParameterDocumentV3 } from './models.js'; +import { BucketDataDocumentV3, BucketParameterDocumentV3, loadBucketDataDocumentV3 } from './models.js'; import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js'; import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; import { MongoCompactorV3 } from './MongoCompactorV3.js'; @@ -365,11 +365,11 @@ export async function* getBucketDataBatchV3( let targetOp: InternalOpId | null = null; for (let rawData of data) { - const row = bucketDataDocumentToTagged( - bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3, - definitionId + const row = loadBucketDataDocumentV3( + { replicationStreamId: ctx.group_id, definitionId }, + bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3 ); - const bucket = row._id.b; + const bucket = row.bucketKey.bucket; if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) { let start: ProtocolOpId | undefined = undefined; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 187be500b..38ad40901 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -66,8 +66,12 @@ export class PersistedBatchV3 extends PersistedBatch { this.debugLastOpId = op_id; this.addBucketDataPut({ + bucketKey: { + bucket: evaluated.bucket, + definitionId: sourceDefinitionId, + replicationStreamId: this.group_id + }, op_id, - definitionId: sourceDefinitionId, bucket: evaluated.bucket, sourceTableId: options.table.id, sourceKey: options.sourceKey, @@ -88,9 +92,12 @@ export class PersistedBatchV3 extends PersistedBatch { } this.addBucketDataRemove({ + bucketKey: { + bucket: bucket.bucket, + definitionId, + replicationStreamId: this.group_id + }, op_id, - definitionId, - bucket: bucket.bucket, sourceTableId: options.table.id, sourceKey: options.sourceKey, table: bucket.table, @@ -254,9 +261,9 @@ export class PersistedBatchV3 extends PersistedBatch { protected async flushBucketData(session: mongo.ClientSession) { const operationsByDefinition = new Map(); for (const document of this.bucketData) { - const existing = operationsByDefinition.get(document.def) ?? []; + const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? []; existing.push(document); - operationsByDefinition.set(document.def, existing); + operationsByDefinition.set(document.bucketKey.definitionId, existing); } for (const [definitionId, documents] of operationsByDefinition.entries()) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts new file mode 100644 index 000000000..95d34fa4c --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts @@ -0,0 +1,82 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { InternalOpId } from '@powersync/service-core'; +import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js'; +import { + BucketDataDocumentGeneric, + BucketDataDocumentGenericId, + SingleBucketStore +} from '../common/SingleBucketStore.js'; +import { BucketDataProperties } from '../models.js'; +import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; +import { BucketDataDocumentV3, BucketDataKeyV3 } from './models.js'; + +export class SingleBucketStoreV3 implements SingleBucketStore { + public readonly collection: mongo.Collection; + + constructor( + private db: VersionedPowerSyncMongoV3, + public readonly key: BucketKey + ) { + this.collection = db.bucketDataV3( + key.replicationStreamId, + key.definitionId + ) as unknown as mongo.Collection; + } + + docId(o: InternalOpId): BucketDataDocumentGenericId { + // `satisfies BucketDataKeyV3` checks that we use the correct type for V3 storage + // `as BucketDataDocumentGenericId` does a cast to get the interface virtual type + return { + b: this.key.bucket, + o + } satisfies BucketDataKeyV3 as BucketDataDocumentGenericId; + } + + get minId(): BucketDataDocumentGenericId { + return { + b: this.key.bucket, + o: new mongo.MinKey() + } as any; // No way to properly type this + } + + get maxId(): BucketDataDocumentGenericId { + return { + b: this.key.bucket, + o: new mongo.MaxKey() + } as any; // No way to properly type this + } + + toPersistedDocument(source: Omit): BucketDataDocumentGeneric { + const { o, ...rest } = source; + const doc: BucketDataDocumentV3 = { + _id: { + b: this.key.bucket, + o: o + }, + ...rest + }; + return doc as BucketDataDocumentGeneric; + } + + fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc { + const document = doc as BucketDataDocumentV3; + const { _id, ...rest } = document; + return { + bucketKey: this.key, + o: _id.o, + ...rest + }; + } + + fromPartialPersistedDocument( + doc: Pick + ): Pick { + const document = doc as Pick; + const { _id, ...rest } = document; + return { + bucketKey: this.key, + o: _id.o, + ...rest + } as Pick; + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index 444562c27..30c10d216 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -1,6 +1,7 @@ import { InternalOpId } from '@powersync/service-core'; import * as bson from 'bson'; import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js'; +import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js'; import { BucketDataDocumentBase, BucketDataKey, @@ -10,7 +11,6 @@ import { ReplicaId, SourceTableDocument, SourceTableKey, - TaggedBucketDataDocument, TaggedBucketParameterDocument } from '../models.js'; @@ -44,9 +44,30 @@ export interface BucketDataDocumentV3 extends BucketDataDocumentBase { _id: BucketDataKeyV3; } -export function taggedBucketDataDocumentToV3(document: TaggedBucketDataDocument): BucketDataDocumentV3 { - const { def: _definitionId, ...rest } = document; - return rest; +export function taggedBucketDataDocumentToV3(document: BucketDataDoc): BucketDataDocumentV3 { + const { bucketKey, o, ...rest } = document; + return { + _id: { + b: bucketKey.bucket, + o: o + }, + ...rest + }; +} + +export function loadBucketDataDocumentV3( + context: Pick, + doc: BucketDataDocumentV3 +): BucketDataDoc { + const { _id, ...rest } = doc; + return { + bucketKey: { + ...context, + bucket: _id.b + }, + o: _id.o, + ...rest + }; } export function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3 { diff --git a/modules/module-mongodb-storage/src/utils/util.ts b/modules/module-mongodb-storage/src/utils/util.ts index f8e8b5334..88f55c6fc 100644 --- a/modules/module-mongodb-storage/src/utils/util.ts +++ b/modules/module-mongodb-storage/src/utils/util.ts @@ -6,7 +6,7 @@ import * as bson from 'bson'; import * as crypto from 'crypto'; import * as timers from 'node:timers/promises'; import * as uuid from 'uuid'; -import { TaggedBucketDataDocument } from '../storage/implementation/models.js'; +import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js'; export function idPrefixFilter(prefix: Partial, rest: (keyof T)[]): mongo.Condition { let filter = { @@ -71,10 +71,10 @@ export async function readSingleBatch(cursor: mongo.AbstractCursor): Promi } } -export function mapOpEntry(row: TaggedBucketDataDocument): utils.OplogEntry { +export function mapOpEntry(row: BucketDataDoc): utils.OplogEntry { if (row.op == 'PUT' || row.op == 'REMOVE') { return { - op_id: utils.internalToExternalOpId(row._id.o), + op_id: utils.internalToExternalOpId(row.o), op: row.op, object_type: row.table, object_id: row.row_id, @@ -86,7 +86,7 @@ export function mapOpEntry(row: TaggedBucketDataDocument): utils.OplogEntry { // MOVE, CLEAR return { - op_id: utils.internalToExternalOpId(row._id.o), + op_id: utils.internalToExternalOpId(row.o), op: row.op, checksum: Number(row.checksum) }; diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index 566597f9c..315fb4bfa 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -23,7 +23,7 @@ import { reduceBucket, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER, StorageDataHelpers, test_utils } from '@powersync/service-core-tests'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as postgres_storage from '@powersync/service-module-postgres-storage'; import * as timers from 'node:timers/promises'; @@ -97,6 +97,7 @@ bucket_definitions: `; const syncRules = await f.updateSyncRules(updateSyncRulesFromYaml(syncRuleContent, { storageVersion })); const storage = f.getInstance(syncRules); + const helpers = new StorageDataHelpers(storage, syncRules); abortController = new AbortController(); const options: WalStreamOptions = { abort_signal: abortController.signal, @@ -182,62 +183,11 @@ bucket_definitions: } const checkpoint = (await storage.getCheckpoint()).checkpoint; - if (f instanceof mongo_storage.storage.MongoBucketStorage) { - const opsBefore = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) - .filter((row) => row._id.o <= checkpoint) - .map((row) => - mongo_storage.storage.bucketDataDocumentToTagged( - row, - mongo_storage.storage.LEGACY_BUCKET_DATA_DEFINITION_ID - ) - ) - .map(mongo_storage.storage.mapOpEntry); - await storage.compact({ maxOpId: checkpoint }); - const opsAfter = (await f.db.bucket_data.find().sort({ _id: 1 }).toArray()) - .filter((row) => row._id.o <= checkpoint) - .map((row) => - mongo_storage.storage.bucketDataDocumentToTagged( - row, - mongo_storage.storage.LEGACY_BUCKET_DATA_DEFINITION_ID - ) - ) - .map(mongo_storage.storage.mapOpEntry); - - test_utils.validateCompactedBucket(opsBefore, opsAfter); - } else if (f instanceof postgres_storage.PostgresBucketStorageFactory) { - const { db } = f; - const opsBefore = ( - await db.sql` - SELECT - * - FROM - bucket_data - WHERE - op_id <= ${{ type: 'int8', value: checkpoint }} - ORDER BY - op_id ASC - ` - .decoded(postgres_storage.models.BucketData) - .rows() - ).map(postgres_storage.utils.mapOpEntry); - await storage.compact({ maxOpId: checkpoint }); - const opsAfter = ( - await db.sql` - SELECT - * - FROM - bucket_data - WHERE - op_id <= ${{ type: 'int8', value: checkpoint }} - ORDER BY - op_id ASC - ` - .decoded(postgres_storage.models.BucketData) - .rows() - ).map(postgres_storage.utils.mapOpEntry); - - test_utils.validateCompactedBucket(opsBefore, opsAfter); - } + const opsBefore = await helpers.getBucketData('global[]', checkpoint); + await storage.compact({ maxOpId: checkpoint }); + const opsAfter = await helpers.getBucketData('global[]', checkpoint); + + test_utils.validateCompactedBucket(opsBefore, opsAfter); } }; @@ -259,24 +209,6 @@ bucket_definitions: return bson.deserialize(doc.data.buffer) as SqliteRow; }); expect(transformed).toEqual([]); - - // Check that each PUT has a REMOVE - const ops = await f.db.bucket_data.find().sort({ _id: 1 }).toArray(); - - // All a single bucket in this test - const bucket = ops - .map((op) => - mongo_storage.storage.bucketDataDocumentToTagged(op, mongo_storage.storage.LEGACY_BUCKET_DATA_DEFINITION_ID) - ) - .map((op) => mongo_storage.storage.mapOpEntry(op)); - const reduced = test_utils.reduceBucket(bucket); - expect(reduced).toMatchObject([ - { - op_id: '0', - op: 'CLEAR' - } - // Should contain no additional data - ]); } else if (f instanceof postgres_storage.storage.PostgresBucketStorageFactory) { const { db } = f; // Check that all inserts have been deleted again @@ -317,6 +249,19 @@ bucket_definitions: // Should contain no additional data ]); } + + // Check that each PUT has a REMOVE + const checkpoint = (await storage.getCheckpoint()).checkpoint; + const ops = await helpers.getBucketData('global[]', checkpoint); + + const reduced = test_utils.reduceBucket(ops); + expect(reduced).toMatchObject([ + { + op_id: '0', + op: 'CLEAR' + } + // Should contain no additional data + ]); } abortController.abort(); diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index 94f1d3f35..381cb167e 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -7,22 +7,19 @@ import { initializeCoreReplicationMetrics, InternalOpId, LEGACY_STORAGE_VERSION, - OplogEntry, settledPromise, storage, - STORAGE_VERSION_CONFIG, SyncRulesBucketStorage, unsettledPromise, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, METRICS_HELPER, StorageDataHelpers, test_utils } from '@powersync/service-core-tests'; import * as pgwire from '@powersync/service-jpgwire'; import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js'; export class WalStreamTestContext implements AsyncDisposable { private _walStream?: WalStream; private abortController = new AbortController(); - private syncRulesId?: number; private syncRulesContent?: storage.PersistedSyncRulesContent; public storage?: SyncRulesBucketStorage; private settledReplicationPromise?: Promise>; @@ -45,17 +42,15 @@ export class WalStreamTestContext implements AsyncDisposable { } const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION; - const versionedBuckets = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false; - return new WalStreamTestContext(f, connectionManager, options?.walStreamOptions, storageVersion, versionedBuckets); + return new WalStreamTestContext(f, connectionManager, options?.walStreamOptions, storageVersion); } constructor( public factory: BucketStorageFactory, public connectionManager: PgManager, private walStreamOptions?: Partial, - private storageVersion: number = LEGACY_STORAGE_VERSION, - private versionedBuckets: boolean = STORAGE_VERSION_CONFIG[storageVersion]?.versionedBuckets ?? false + private storageVersion: number = LEGACY_STORAGE_VERSION ) { createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); @@ -97,7 +92,6 @@ export class WalStreamTestContext implements AsyncDisposable { const syncRules = await this.factory.updateSyncRules( updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) ); - this.syncRulesId = syncRules.id; this.syncRulesContent = syncRules; this.storage = this.factory.getInstance(syncRules); return this.storage!; @@ -109,7 +103,6 @@ export class WalStreamTestContext implements AsyncDisposable { throw new Error(`Next sync rules not available`); } - this.syncRulesId = syncRules.id; this.syncRulesContent = syncRules; this.storage = this.factory.getInstance(syncRules); return this.storage!; @@ -121,7 +114,6 @@ export class WalStreamTestContext implements AsyncDisposable { throw new Error(`Active sync rules not available`); } - this.syncRulesId = syncRules.id; this.syncRulesContent = syncRules; this.storage = this.factory.getInstance(syncRules); return this.storage!; @@ -194,35 +186,18 @@ export class WalStreamTestContext implements AsyncDisposable { } async getBucketsDataBatch(buckets: Record, options?: { timeout?: number }) { - let checkpoint = await this.getCheckpoint(options); - const syncRules = this.getSyncRulesContent(); - const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(syncRules, bucket, start)); - return test_utils.fromAsync(this.storage!.getBucketDataBatch(checkpoint, map)); + const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent()); + const checkpoint = await this.getCheckpoint(options); + return helpers.getBucketsDataBatch(buckets, checkpoint); } /** * This waits for a client checkpoint. */ async getBucketData(bucket: string, start?: InternalOpId | string | undefined, options?: { timeout?: number }) { - start ??= 0n; - if (typeof start == 'string') { - start = BigInt(start); - } - const syncRules = this.getSyncRulesContent(); + const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent()); const checkpoint = await this.getCheckpoint(options); - let map = [bucketRequest(syncRules, bucket, start)]; - let data: OplogEntry[] = []; - while (true) { - const batch = this.storage!.getBucketDataBatch(checkpoint, map); - - const batches = await test_utils.fromAsync(batch); - data = data.concat(batches[0]?.chunkData.data ?? []); - if (batches.length == 0 || !batches[0]!.chunkData.has_more) { - break; - } - map = [bucketRequest(syncRules, bucket, BigInt(batches[0]!.chunkData.next_after))]; - } - return data; + return helpers.getBucketData(bucket, checkpoint, start); } async getChecksums(buckets: string[], options?: { timeout?: number }) { diff --git a/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts b/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts new file mode 100644 index 000000000..20363e5ea --- /dev/null +++ b/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts @@ -0,0 +1,44 @@ +import { + InternalOpId, + OplogEntry, + PersistedSyncRules, + PersistedSyncRulesContent, + SyncRulesBucketStorage +} from '@powersync/service-core'; +import { bucketRequest } from './general-utils.js'; +import { fromAsync } from './stream_utils.js'; + +export class StorageDataHelpers { + storage: SyncRulesBucketStorage; + syncRules: PersistedSyncRulesContent | PersistedSyncRules; + + constructor(storage: SyncRulesBucketStorage, syncRules: PersistedSyncRulesContent | PersistedSyncRules) { + this.storage = storage; + this.syncRules = syncRules; + } + + async getBucketData(bucket: string, checkpoint: InternalOpId, start?: InternalOpId | string | undefined) { + start ??= 0n; + if (typeof start == 'string') { + start = BigInt(start); + } + let map = [bucketRequest(this.syncRules, bucket, start)]; + let data: OplogEntry[] = []; + while (true) { + const batch = this.storage!.getBucketDataBatch(checkpoint, map); + + const batches = await fromAsync(batch); + data = data.concat(batches[0]?.chunkData.data ?? []); + if (batches.length == 0 || !batches[0]!.chunkData.has_more) { + break; + } + map = [bucketRequest(this.syncRules, bucket, BigInt(batches[0]!.chunkData.next_after))]; + } + return data; + } + + async getBucketsDataBatch(buckets: Record, checkpoint: InternalOpId) { + const map = Object.entries(buckets).map(([bucket, start]) => bucketRequest(this.syncRules, bucket, start)); + return fromAsync(this.storage!.getBucketDataBatch(checkpoint, map)); + } +} diff --git a/packages/service-core-tests/src/test-utils/test-utils-index.ts b/packages/service-core-tests/src/test-utils/test-utils-index.ts index 1b174d84c..a79c44098 100644 --- a/packages/service-core-tests/src/test-utils/test-utils-index.ts +++ b/packages/service-core-tests/src/test-utils/test-utils-index.ts @@ -1,4 +1,5 @@ export * from './bucket-validation.js'; export * from './general-utils.js'; export * from './MetricsHelper.js'; +export * from './StorageDataHelpers.js'; export * from './stream_utils.js'; From 08468e7de8201a9bc011e77ff8a1483db4925ead Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 16:47:37 +0200 Subject: [PATCH 89/93] Minor cleanup. --- .../storage/implementation/MongoChecksums.ts | 15 --------------- .../implementation/v3/MongoChecksumsV3.ts | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index 6a81ce95c..f12d2c722 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -307,21 +307,6 @@ export abstract class MongoChecksums { } } -export function createV3BucketFilter(request: Pick) { - return { - _id: { - $gt: { - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - b: request.bucket, - o: request.end - } - } - }; -} - export function emptyChecksumForRequest( request: Pick ): PartialOrFullChecksum { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index 7c5bd5af1..88e0b42fa 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -1,4 +1,5 @@ import { + bson, BucketChecksum, FetchPartialBucketChecksum, InternalOpId, @@ -7,7 +8,6 @@ import { } from '@powersync/service-core'; import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; import { - createV3BucketFilter, emptyChecksumForRequest, FetchPartialBucketChecksumV3, MongoChecksumOptions, @@ -103,3 +103,18 @@ export class MongoChecksumsV3 extends MongoChecksums { return this.computePartialChecksumsDirectByDefinition(this.normalizeBatch(batch)); } } + +function createV3BucketFilter(request: Pick) { + return { + _id: { + $gt: { + b: request.bucket, + o: request.start ?? new bson.MinKey() + }, + $lte: { + b: request.bucket, + o: request.end + } + } + }; +} From 753d50c1d97410feb814b4fd3a8d7f23b4de62fd Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 17:05:04 +0200 Subject: [PATCH 90/93] Explicitly list fields when persisting. --- .../implementation/v1/PersistedBatchV1.ts | 4 ++-- .../implementation/v1/SingleBucketStoreV1.ts | 13 ++---------- .../src/storage/implementation/v1/models.ts | 14 ++++++++++--- .../implementation/v3/PersistedBatchV3.ts | 4 ++-- .../implementation/v3/SingleBucketStoreV3.ts | 20 +++---------------- .../src/storage/implementation/v3/models.ts | 14 ++++++++++--- 6 files changed, 31 insertions(+), 38 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index 3a2f571fa..84324bca3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -19,7 +19,7 @@ import { BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument, - taggedBucketDataDocumentToV1, + serializeBucketDataV1, taggedBucketParameterDocumentToV1 } from './models.js'; @@ -213,7 +213,7 @@ export class PersistedBatchV1 extends PersistedBatch { await this.db.bucketDataV1.bulkWrite( this.bucketData.map((document) => ({ insertOne: { - document: taggedBucketDataDocumentToV1(document) + document: serializeBucketDataV1(document) } })), { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts index dac3d7f61..c0300bc16 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/SingleBucketStoreV1.ts @@ -8,7 +8,7 @@ import { } from '../common/SingleBucketStore.js'; import { BucketDataProperties } from '../models.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; -import { BucketDataDocumentV1, BucketDataKeyV1 } from './models.js'; +import { BucketDataDocumentV1, BucketDataKeyV1, serializeBucketDataV1 } from './models.js'; export class SingleBucketStoreV1 implements SingleBucketStore { public readonly collection: mongo.Collection; @@ -47,16 +47,7 @@ export class SingleBucketStoreV1 implements SingleBucketStore { } toPersistedDocument(source: Omit): BucketDataDocumentGeneric { - const { o, ...rest } = source; - const doc: BucketDataDocumentV1 = { - _id: { - g: this.key.replicationStreamId, - b: this.key.bucket, - o: o - }, - ...rest - }; - return doc as unknown as BucketDataDocumentGeneric; + return serializeBucketDataV1({ bucketKey: this.key, ...source }) as unknown as BucketDataDocumentGeneric; } fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts index 73711fd26..e5567ce4c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/models.ts @@ -33,15 +33,23 @@ export interface BucketDataDocumentV1 extends BucketDataDocumentBase { _id: BucketDataKeyV1; } -export function taggedBucketDataDocumentToV1(document: BucketDataDoc): BucketDataDocumentV1 { - const { bucketKey, o, ...rest } = document; +export function serializeBucketDataV1(document: BucketDataDoc): BucketDataDocumentV1 { + const { bucketKey, o } = document; return { _id: { g: bucketKey.replicationStreamId, b: bucketKey.bucket, o: o }, - ...rest + // List fields directly, so that we don't accidentally persist any unknown fields + op: document.op, + source_table: document.source_table, + source_key: document.source_key, + table: document.table, + row_id: document.row_id, + checksum: document.checksum, + data: document.data, + target_op: document.target_op }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 38ad40901..7b93a0314 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -18,8 +18,8 @@ import { BucketParameterDocumentV3, BucketStateDocumentV3, CurrentDataDocumentV3, + serializeBucketDataV3, SourceTableDocumentV3, - taggedBucketDataDocumentToV3, taggedBucketParameterDocumentToV3 } from './models.js'; import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; @@ -270,7 +270,7 @@ export class PersistedBatchV3 extends PersistedBatch { await this.db.bucketDataV3(this.group_id, definitionId).bulkWrite( documents.map((document) => ({ insertOne: { - document: taggedBucketDataDocumentToV3(document) + document: serializeBucketDataV3(document) } })), { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts index 95d34fa4c..037773723 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/SingleBucketStoreV3.ts @@ -8,7 +8,7 @@ import { } from '../common/SingleBucketStore.js'; import { BucketDataProperties } from '../models.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; -import { BucketDataDocumentV3, BucketDataKeyV3 } from './models.js'; +import { BucketDataDocumentV3, BucketDataKeyV3, loadBucketDataDocumentV3, serializeBucketDataV3 } from './models.js'; export class SingleBucketStoreV3 implements SingleBucketStore { public readonly collection: mongo.Collection; @@ -47,25 +47,11 @@ export class SingleBucketStoreV3 implements SingleBucketStore { } toPersistedDocument(source: Omit): BucketDataDocumentGeneric { - const { o, ...rest } = source; - const doc: BucketDataDocumentV3 = { - _id: { - b: this.key.bucket, - o: o - }, - ...rest - }; - return doc as BucketDataDocumentGeneric; + return serializeBucketDataV3({ bucketKey: this.key, ...source }) as BucketDataDocumentGeneric; } fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc { - const document = doc as BucketDataDocumentV3; - const { _id, ...rest } = document; - return { - bucketKey: this.key, - o: _id.o, - ...rest - }; + return loadBucketDataDocumentV3(this.key, doc as BucketDataDocumentV3); } fromPartialPersistedDocument( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts index 30c10d216..64f0c5516 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/models.ts @@ -44,14 +44,22 @@ export interface BucketDataDocumentV3 extends BucketDataDocumentBase { _id: BucketDataKeyV3; } -export function taggedBucketDataDocumentToV3(document: BucketDataDoc): BucketDataDocumentV3 { - const { bucketKey, o, ...rest } = document; +export function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3 { + const { bucketKey, o } = document; return { _id: { b: bucketKey.bucket, o: o }, - ...rest + // List fields directly, so that we don't accidentally persist any unknown fields + op: document.op, + source_table: document.source_table, + source_key: document.source_key, + table: document.table, + row_id: document.row_id, + checksum: document.checksum, + data: document.data, + target_op: document.target_op }; } From 8d8164f81312460c3d421df5dcc189115df7c1ef Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 17:20:38 +0200 Subject: [PATCH 91/93] Merge implementations for saveBucketData. --- .../implementation/common/PersistedBatch.ts | 89 ++++++++++++++++--- .../implementation/v1/PersistedBatchV1.ts | 83 ++--------------- .../implementation/v3/PersistedBatchV3.ts | 87 ++---------------- 3 files changed, 93 insertions(+), 166 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 485d3788f..8607ec533 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -2,10 +2,12 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; -import { InternalOpId, storage } from '@powersync/service-core'; -import { mongoTableId } from '../../../utils/util.js'; +import { logger as defaultLogger, Logger } from '@powersync/lib-services-framework'; +import { InternalOpId, storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; import { BucketDefinitionId, BucketDefinitionMapping } from '../BucketDefinitionMapping.js'; +import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; import { MongoIdSequence } from '../MongoIdSequence.js'; import type { VersionedPowerSyncMongo } from '../db.js'; import { TaggedBucketParameterDocument } from '../models.js'; @@ -94,7 +96,77 @@ export abstract class PersistedBatch { this.logger = options?.logger ?? defaultLogger; } - abstract saveBucketData(options: SaveBucketDataOptions): void; + saveBucketData(options: SaveBucketDataOptions) { + const remaining_buckets = new Map(); + for (let bucket of options.before_buckets) { + remaining_buckets.set(currentBucketKey(bucket), bucket); + } + + const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); + + for (const evaluated of options.evaluated) { + const sourceDefinitionId = this.mapping.bucketSourceId(evaluated.source); + const key = currentBucketKey({ + definitionId: sourceDefinitionId, + bucket: evaluated.bucket, + table: evaluated.table, + id: evaluated.id + }); + + const recordData = JSONBig.stringify(evaluated.data); + const checksum = utils.hashData(evaluated.table, evaluated.id, recordData); + if (recordData.length > MAX_ROW_SIZE) { + this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`); + continue; + } + + remaining_buckets.delete(key); + const byteEstimate = recordData.length + 200; + this.currentSize += byteEstimate; + + const op_id = options.op_seq.next(); + this.debugLastOpId = op_id; + + this.addBucketDataPut({ + bucketKey: { + bucket: evaluated.bucket, + definitionId: sourceDefinitionId, + replicationStreamId: this.group_id + }, + op_id, + bucket: evaluated.bucket, + sourceTableId: options.table.id, + sourceKey: options.sourceKey, + table: evaluated.table, + rowId: evaluated.id, + checksum: BigInt(checksum), + data: recordData + }); + this.incrementBucket(sourceDefinitionId, evaluated.bucket, op_id, byteEstimate); + } + + for (let bucket of remaining_buckets.values()) { + const definitionId = this.checkDefinitionId(bucket.definitionId); + const op_id = options.op_seq.next(); + this.debugLastOpId = op_id; + + this.addBucketDataRemove({ + bucketKey: { + replicationStreamId: this.group_id, + definitionId, + bucket: bucket.bucket + }, + op_id, + sourceTableId: options.table.id, + sourceKey: options.sourceKey, + table: bucket.table, + rowId: bucket.id, + checksum: dchecksum + }); + this.currentSize += 200; + this.incrementBucket(definitionId, bucket.bucket, op_id, 200); + } + } abstract saveParameterData(data: SaveParameterDataOptions): void; @@ -120,16 +192,13 @@ export abstract class PersistedBatch { protected abstract resetCurrentData(): void; + protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId; + protected get bucketDataCount(): number { return this.bucketData.length; } - protected incrementBucket( - definitionId: BucketDefinitionId | null, - bucket: string, - op_id: InternalOpId, - bytes: number - ) { + protected incrementBucket(definitionId: BucketDefinitionId, bucket: string, op_id: InternalOpId, bytes: number) { const key = `${definitionId ?? ''}:${bucket}`; let existingState = this.bucketStates.get(key); if (existingState) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index 84324bca3..3e23c2541 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -1,15 +1,14 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; -import { storage, utils } from '@powersync/service-core'; -import { JSONBig } from '@powersync/service-jsonbig'; +import { storage } from '@powersync/service-core'; import * as bson from 'bson'; -import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; -import { currentBucketKey, EMPTY_DATA, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; +import { mongoTableId } from '../../../utils/util.js'; +import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; +import { EMPTY_DATA } from '../MongoBucketBatchShared.js'; import { BucketStateUpdate, PersistedBatch, - SaveBucketDataOptions, SaveParameterDataOptions, UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; @@ -28,77 +27,9 @@ export class PersistedBatchV1 extends PersistedBatch { currentData: mongo.AnyBulkWriteOperation[] = []; - saveBucketData(options: SaveBucketDataOptions) { - const remaining_buckets = new Map(); - for (let bucket of options.before_buckets) { - if (bucket.definitionId != null) { - throw new ReplicationAssertionError('Unexpected v3 bucket when incrementalReprocessing is disabled'); - } - remaining_buckets.set(currentBucketKey(bucket), bucket); - } - - const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); - - for (const evaluated of options.evaluated) { - const key = currentBucketKey({ - definitionId: null, - bucket: evaluated.bucket, - table: evaluated.table, - id: evaluated.id - }); - - const recordData = JSONBig.stringify(evaluated.data); - const checksum = utils.hashData(evaluated.table, evaluated.id, recordData); - if (recordData.length > MAX_ROW_SIZE) { - this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`); - continue; - } - - remaining_buckets.delete(key); - const byteEstimate = recordData.length + 200; - this.currentSize += byteEstimate; - - const op_id = options.op_seq.next(); - this.debugLastOpId = op_id; - - this.addBucketDataPut({ - bucketKey: { - replicationStreamId: this.group_id, - definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, - bucket: evaluated.bucket - }, - op_id, - bucket: evaluated.bucket, - sourceTableId: options.table.id, - sourceKey: options.sourceKey, - table: evaluated.table, - rowId: evaluated.id, - checksum: BigInt(checksum), - data: recordData - }); - this.incrementBucket(null, evaluated.bucket, op_id, byteEstimate); - } - - for (let bucket of remaining_buckets.values()) { - const op_id = options.op_seq.next(); - this.debugLastOpId = op_id; - - this.addBucketDataRemove({ - bucketKey: { - replicationStreamId: this.group_id, - definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID, - bucket: bucket.bucket - }, - op_id, - sourceTableId: options.table.id, - sourceKey: options.sourceKey, - table: bucket.table, - rowId: bucket.id, - checksum: dchecksum - }); - this.currentSize += 200; - this.incrementBucket(null, bucket.bucket, op_id, 200); - } + protected checkDefinitionId(_definitionId: BucketDefinitionId | null): BucketDefinitionId { + // V1 storage doesn't persist the id, and we don't use it. + return LEGACY_BUCKET_DATA_DEFINITION_ID; } saveParameterData(data: SaveParameterDataOptions) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 7b93a0314..9c623cc72 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -1,19 +1,16 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; -import { InternalOpId, storage, utils } from '@powersync/service-core'; -import { JSONBig } from '@powersync/service-jsonbig'; +import { InternalOpId, storage } from '@powersync/service-core'; import * as bson from 'bson'; -import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js'; +import { mongoTableId } from '../../../utils/util.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { BucketStateUpdate, PersistedBatch, - SaveBucketDataOptions, SaveParameterDataOptions, UpsertCurrentDataOptions } from '../common/PersistedBatch.js'; import { SourceTableKey } from '../models.js'; -import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js'; import { BucketParameterDocumentV3, BucketStateDocumentV3, @@ -31,82 +28,12 @@ export class PersistedBatchV3 extends PersistedBatch { currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation }[] = []; sourceTablePendingDeletes = new Map(); - saveBucketData(options: SaveBucketDataOptions) { - const remaining_buckets = new Map(); - for (let bucket of options.before_buckets) { - if (bucket.definitionId == null) { - throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); - } - remaining_buckets.set(currentBucketKey(bucket), bucket); - } - - const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); - - for (const evaluated of options.evaluated) { - const sourceDefinitionId = this.mapping.bucketSourceId(evaluated.source); - const key = currentBucketKey({ - definitionId: sourceDefinitionId, - bucket: evaluated.bucket, - table: evaluated.table, - id: evaluated.id - }); - - const recordData = JSONBig.stringify(evaluated.data); - const checksum = utils.hashData(evaluated.table, evaluated.id, recordData); - if (recordData.length > MAX_ROW_SIZE) { - this.logger.error(`Row ${key} too large: ${recordData.length} bytes. Removing.`); - continue; - } - - remaining_buckets.delete(key); - const byteEstimate = recordData.length + 200; - this.currentSize += byteEstimate; - - const op_id = options.op_seq.next(); - this.debugLastOpId = op_id; - - this.addBucketDataPut({ - bucketKey: { - bucket: evaluated.bucket, - definitionId: sourceDefinitionId, - replicationStreamId: this.group_id - }, - op_id, - bucket: evaluated.bucket, - sourceTableId: options.table.id, - sourceKey: options.sourceKey, - table: evaluated.table, - rowId: evaluated.id, - checksum: BigInt(checksum), - data: recordData - }); - this.incrementBucket(sourceDefinitionId, evaluated.bucket, op_id, byteEstimate); - } - - for (let bucket of remaining_buckets.values()) { - const op_id = options.op_seq.next(); - this.debugLastOpId = op_id; - const definitionId = bucket.definitionId; - if (definitionId == null) { - throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); - } - - this.addBucketDataRemove({ - bucketKey: { - bucket: bucket.bucket, - definitionId, - replicationStreamId: this.group_id - }, - op_id, - sourceTableId: options.table.id, - sourceKey: options.sourceKey, - table: bucket.table, - rowId: bucket.id, - checksum: dchecksum - }); - this.currentSize += 200; - this.incrementBucket(definitionId, bucket.bucket, op_id, 200); + protected checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId { + if (definitionId == null) { + // This is required for V3 storage. + throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled'); } + return definitionId; } saveParameterData(data: SaveParameterDataOptions) { From 1b90f98f5e6941d49e292795e891bf33d018a7f6 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Thu, 2 Apr 2026 17:31:42 +0200 Subject: [PATCH 92/93] Fix regression. --- .../implementation/common/PersistedBatch.ts | 21 ++++++++++++------- .../implementation/v1/PersistedBatchV1.ts | 5 +++++ .../implementation/v3/PersistedBatchV3.ts | 5 +++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts index 8607ec533..b7da3c237 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/common/PersistedBatch.ts @@ -1,5 +1,5 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; +import { BucketDataSource, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { logger as defaultLogger, Logger } from '@powersync/lib-services-framework'; @@ -99,15 +99,21 @@ export abstract class PersistedBatch { saveBucketData(options: SaveBucketDataOptions) { const remaining_buckets = new Map(); for (let bucket of options.before_buckets) { - remaining_buckets.set(currentBucketKey(bucket), bucket); + const mapped: SourceRecordBucketState = { + bucket: bucket.bucket, + definitionId: this.checkDefinitionId(bucket.definitionId), + id: bucket.id, + table: bucket.table + }; + remaining_buckets.set(currentBucketKey(mapped), mapped); } const dchecksum = BigInt(utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey))); for (const evaluated of options.evaluated) { - const sourceDefinitionId = this.mapping.bucketSourceId(evaluated.source); + const definitionId = this.getBucketDefinitionId(evaluated.source); const key = currentBucketKey({ - definitionId: sourceDefinitionId, + definitionId: definitionId, bucket: evaluated.bucket, table: evaluated.table, id: evaluated.id @@ -130,7 +136,7 @@ export abstract class PersistedBatch { this.addBucketDataPut({ bucketKey: { bucket: evaluated.bucket, - definitionId: sourceDefinitionId, + definitionId: definitionId, replicationStreamId: this.group_id }, op_id, @@ -142,11 +148,11 @@ export abstract class PersistedBatch { checksum: BigInt(checksum), data: recordData }); - this.incrementBucket(sourceDefinitionId, evaluated.bucket, op_id, byteEstimate); + this.incrementBucket(definitionId, evaluated.bucket, op_id, byteEstimate); } for (let bucket of remaining_buckets.values()) { - const definitionId = this.checkDefinitionId(bucket.definitionId); + const definitionId = bucket.definitionId!; const op_id = options.op_seq.next(); this.debugLastOpId = op_id; @@ -193,6 +199,7 @@ export abstract class PersistedBatch { protected abstract resetCurrentData(): void; protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId; + protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId; protected get bucketDataCount(): number { return this.bucketData.length; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts index 3e23c2541..e300bc29a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/PersistedBatchV1.ts @@ -3,6 +3,7 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import * as bson from 'bson'; +import { BucketDataSource } from '@powersync/service-sync-rules'; import { mongoTableId } from '../../../utils/util.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; import { EMPTY_DATA } from '../MongoBucketBatchShared.js'; @@ -32,6 +33,10 @@ export class PersistedBatchV1 extends PersistedBatch { return LEGACY_BUCKET_DATA_DEFINITION_ID; } + protected getBucketDefinitionId(_bucketSource: BucketDataSource): BucketDefinitionId { + return LEGACY_BUCKET_DATA_DEFINITION_ID; + } + saveParameterData(data: SaveParameterDataOptions) { const { sourceTable, sourceKey, evaluated } = data; const remaining_lookups = new Map(); diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts index 9c623cc72..e3be5e8f0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/PersistedBatchV3.ts @@ -1,6 +1,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { InternalOpId, storage } from '@powersync/service-core'; +import { BucketDataSource } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { mongoTableId } from '../../../utils/util.js'; import { BucketDefinitionId } from '../BucketDefinitionMapping.js'; @@ -36,6 +37,10 @@ export class PersistedBatchV3 extends PersistedBatch { return definitionId; } + protected getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId { + return this.mapping.bucketSourceId(bucketSource); + } + saveParameterData(data: SaveParameterDataOptions) { const { sourceTable, sourceKey, evaluated } = data; const remaining_lookups = new Map(); From 13992be078468f2fd4f6c5f73971c4041df4ae9f Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Fri, 10 Apr 2026 11:50:05 +0200 Subject: [PATCH 93/93] Avoid double-parsing sync rules in many cases when updating. --- .../src/storage/MongoBucketStorage.ts | 6 +----- .../service-core/src/storage/BucketStorageFactory.ts | 11 +++++++++-- .../src/storage/PersistedSyncRulesContent.ts | 4 +++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index 69dd7a1c5..07a426a4d 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -1,7 +1,6 @@ import { GetIntanceOptions, storage } from '@powersync/service-core'; import { DO_NOT_LOG, ErrorCode, ServiceError } from '@powersync/lib-services-framework'; -import { SqlSyncRules } from '@powersync/service-sync-rules'; import { v4 as uuid } from 'uuid'; import * as lib_mongo from '@powersync/lib-service-mongodb'; @@ -208,10 +207,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { last_keepalive_ts: null }; if (storageConfig.incrementalReprocessing) { - const parsed = SqlSyncRules.fromYaml(options.config.yaml, { - schema: undefined, - defaultSchema: 'not_applicable' - }); + const parsed = options.config.parsed; doc.rule_mapping = BucketDefinitionMapping.fromParsedSyncRules(parsed).serialize(); } await this.db.sync_rules.insertOne(doc); diff --git a/packages/service-core/src/storage/BucketStorageFactory.ts b/packages/service-core/src/storage/BucketStorageFactory.ts index 6b68aee45..79a2eb9c0 100644 --- a/packages/service-core/src/storage/BucketStorageFactory.ts +++ b/packages/service-core/src/storage/BucketStorageFactory.ts @@ -161,6 +161,12 @@ export interface UpdateSyncRulesOptions { * compiler. */ plan: SerializedSyncPlan | null; + + /** + * Parsed sync rules version, primarily to generate a definition mapping. + * Not persisted, and the defaultSchema used for parsing is not relevant. + */ + parsed: SyncConfigWithErrors; }; lock?: boolean; storageVersion?: number; @@ -198,10 +204,11 @@ export function updateSyncRulesFromYaml( } export function updateSyncRulesFromConfig( - { config, errors }: SyncConfigWithErrors, + parsed: SyncConfigWithErrors, options?: Omit ): UpdateSyncRulesOptions { let plan: SerializedSyncPlan | null = null; + const { config, errors } = parsed; if (config instanceof PrecompiledSyncConfig) { const eventDescriptors: Record = {}; for (const event of config.eventDescriptors) { @@ -216,7 +223,7 @@ export function updateSyncRulesFromConfig( }; } - return { config: { yaml: config.content, plan }, ...options }; + return { config: { yaml: config.content, plan, parsed }, ...options }; } export interface GetIntanceOptions { diff --git a/packages/service-core/src/storage/PersistedSyncRulesContent.ts b/packages/service-core/src/storage/PersistedSyncRulesContent.ts index 97716b1ba..52de3b457 100644 --- a/packages/service-core/src/storage/PersistedSyncRulesContent.ts +++ b/packages/service-core/src/storage/PersistedSyncRulesContent.ts @@ -144,8 +144,10 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon } asUpdateOptions(options?: Omit): UpdateSyncRulesOptions { + // defaultSchema is not relevant for the parsed version here + const parsed = this.parsed({ defaultSchema: 'not_applicable' }); return { - config: { yaml: this.sync_rules_content, plan: this.compiled_plan }, + config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed: parsed.sync_rules }, ...options }; }