From 880aed392d232d2d1516499ddf73307258f6600a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 12 Jun 2026 13:30:00 +0200 Subject: [PATCH 01/11] experiment: mikro sqlite storage --- modules/module-mikroorm-storage/package.json | 52 + .../sqlite/SqliteMigrationLockManager.ts | 89 + .../sqlite/SqliteMikroOrmStorageFactory.ts | 18 + .../src/drivers/sqlite/sqlite-config.ts | 26 + .../src/drivers/sqlite/sqlite-dialect.ts | 92 + .../src/entities/common/bucket-data.schema.ts | 46 + .../common/bucket-parameters.schema.ts | 34 + .../entities/common/current-data.schema.ts | 38 + .../src/entities/common/instance.schema.ts | 14 + .../entities/common/source-table.schema.ts | 41 + .../src/entities/common/sync-rules.schema.ts | 46 + .../common/write-checkpoint.schema.ts | 30 + .../src/entities/entities-index.ts | 7 + modules/module-mikroorm-storage/src/index.ts | 10 + .../AbstractMikroOrmMigrationLockManager.ts | 101 + .../src/migrations/MikroOrmMigrationAgent.ts | 93 + .../src/migrations/NoOpMigrationStore.ts | 15 + ...tion20260612083946_InitialSqliteStorage.ts | 53 + .../src/mikro-orm.config.ts | 43 + .../src/module/MikroOrmStorageModule.ts | 26 + .../src/storage/MikroOrmBucketBatch.ts | 637 +++ .../storage/MikroOrmBucketStorageFactory.ts | 247 ++ .../src/storage/MikroOrmCompactor.ts | 498 +++ .../src/storage/MikroOrmPersistedBatch.ts | 375 ++ .../MikroOrmPersistedReplicationStream.ts | 81 + .../src/storage/MikroOrmReportStorage.ts | 34 + .../src/storage/MikroOrmStorageDialect.ts | 65 + .../src/storage/MikroOrmStorageProvider.ts | 70 + .../src/storage/MikroOrmSyncRulesStorage.ts | 594 +++ .../src/storage/storage-index.ts | 8 + .../src/storage/unsupported.ts | 3 + .../src/types/types.ts | 45 + .../src/__snapshots__/storage.test.ts.snap | 124 + .../__snapshots__/storage_sync.test.ts.snap | 3568 +++++++++++++++++ .../test/src/migrations.test.ts | 174 + .../module-mikroorm-storage/test/src/setup.ts | 11 + .../test/src/storage-provider.test.ts | 38 + .../test/src/storage.test.ts | 21 + .../test/src/storage_sync.test.ts | 14 + .../test/src/sync-rules-storage.test.ts | 99 + .../module-mikroorm-storage/test/src/util.ts | 26 + .../test/tsconfig.json | 18 + modules/module-mikroorm-storage/tsconfig.json | 25 + .../module-mikroorm-storage/vitest.config.ts | 7 + packages/schema/package.json | 1 + .../schema/src/scripts/compile-json-schema.ts | 5 +- .../service-core/src/storage/StorageEngine.ts | 4 +- .../src/storage/StorageProvider.ts | 1 + .../service-core/src/system/ServiceContext.ts | 3 +- plans/mikroorm-bucket-storage.md | 291 ++ pnpm-lock.yaml | 144 + pnpm-workspace.yaml | 2 + service/package.json | 1 + service/src/util/modules.ts | 2 + tsconfig.json | 6 + 55 files changed, 8113 insertions(+), 3 deletions(-) create mode 100644 modules/module-mikroorm-storage/package.json create mode 100644 modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts create mode 100644 modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMikroOrmStorageFactory.ts create mode 100644 modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts create mode 100644 modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts create mode 100644 modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts create mode 100644 modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts create mode 100644 modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts create mode 100644 modules/module-mikroorm-storage/src/entities/common/instance.schema.ts create mode 100644 modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts create mode 100644 modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts create mode 100644 modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts create mode 100644 modules/module-mikroorm-storage/src/entities/entities-index.ts create mode 100644 modules/module-mikroorm-storage/src/index.ts create mode 100644 modules/module-mikroorm-storage/src/migrations/AbstractMikroOrmMigrationLockManager.ts create mode 100644 modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts create mode 100644 modules/module-mikroorm-storage/src/migrations/NoOpMigrationStore.ts create mode 100644 modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612083946_InitialSqliteStorage.ts create mode 100644 modules/module-mikroorm-storage/src/mikro-orm.config.ts create mode 100644 modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts create mode 100644 modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts create mode 100644 modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts create mode 100644 modules/module-mikroorm-storage/src/storage/MikroOrmCompactor.ts create mode 100644 modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts create mode 100644 modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts create mode 100644 modules/module-mikroorm-storage/src/storage/MikroOrmReportStorage.ts create mode 100644 modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts create mode 100644 modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts create mode 100644 modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts create mode 100644 modules/module-mikroorm-storage/src/storage/storage-index.ts create mode 100644 modules/module-mikroorm-storage/src/storage/unsupported.ts create mode 100644 modules/module-mikroorm-storage/src/types/types.ts create mode 100644 modules/module-mikroorm-storage/test/src/__snapshots__/storage.test.ts.snap create mode 100644 modules/module-mikroorm-storage/test/src/__snapshots__/storage_sync.test.ts.snap create mode 100644 modules/module-mikroorm-storage/test/src/migrations.test.ts create mode 100644 modules/module-mikroorm-storage/test/src/setup.ts create mode 100644 modules/module-mikroorm-storage/test/src/storage-provider.test.ts create mode 100644 modules/module-mikroorm-storage/test/src/storage.test.ts create mode 100644 modules/module-mikroorm-storage/test/src/storage_sync.test.ts create mode 100644 modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts create mode 100644 modules/module-mikroorm-storage/test/src/util.ts create mode 100644 modules/module-mikroorm-storage/test/tsconfig.json create mode 100644 modules/module-mikroorm-storage/tsconfig.json create mode 100644 modules/module-mikroorm-storage/vitest.config.ts create mode 100644 plans/mikroorm-bucket-storage.md diff --git a/modules/module-mikroorm-storage/package.json b/modules/module-mikroorm-storage/package.json new file mode 100644 index 000000000..34197788e --- /dev/null +++ b/modules/module-mikroorm-storage/package.json @@ -0,0 +1,52 @@ +{ + "name": "@powersync/service-module-mikroorm-storage", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/index.d.ts", + "version": "0.1.0", + "main": "dist/index.js", + "license": "FSL-1.1-ALv2", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc -b", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./dist && tsc -b --clean", + "mikroorm:generate-migration": "pnpm build && mikro-orm migration:create --config dist/mikro-orm.config.js", + "mikroorm:generate-migration:sqlite": "MIKRO_ORM_STORAGE_DIALECT=sqlite pnpm mikroorm:generate-migration", + "test": "vitest" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "types": "./dist/types/types.d.ts", + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js" + } + }, + "dependencies": { + "@mikro-orm/core": "^7.1.4", + "@mikro-orm/migrations": "^7.1.4", + "@mikro-orm/sql": "^7.1.4", + "@mikro-orm/sqlite": "^7.1.4", + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-core": "workspace:*", + "@powersync/service-jsonbig": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "ts-codec": "^1.3.0", + "uuid": "catalog:" + }, + "devDependencies": { + "@mikro-orm/cli": "^7.1.4", + "@powersync/service-core-tests": "workspace:*", + "typescript": "catalog:" + } +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts new file mode 100644 index 000000000..d20030940 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts @@ -0,0 +1,89 @@ +import { AbstractMikroOrmMigrationLockManager } from '../../migrations/AbstractMikroOrmMigrationLockManager.js'; + +/** + * SQLite implementation of the migration lock. + * + * This intentionally uses raw SQLite statements instead of MikroORM entities. Migration locking has a classic + * chicken-and-egg problem: the service must take a DB-backed lock before it asks MikroORM to run the migrations + * that normally create and evolve application tables. The lock table therefore has to bootstrap itself with + * `CREATE TABLE IF NOT EXISTS`, outside the generated MikroORM migration model. + */ +export class SqliteMigrationLockManager extends AbstractMikroOrmMigrationLockManager { + protected async initLockStore(): Promise { + await this.execute( + ` + CREATE TABLE IF NOT EXISTS powersync_migration_locks ( + name TEXT PRIMARY KEY, + lock_id TEXT, + expires_at INTEGER NOT NULL + ) + `, + [] + ); + + await this.execute( + ` + INSERT OR IGNORE INTO powersync_migration_locks (name, lock_id, expires_at) + VALUES (?, NULL, 0) + `, + [this.name] + ); + } + + protected async tryAcquireLock(options: { + name: string; + lockId: string; + now: Date; + expiresAt: Date; + }): Promise { + const result = await this.execute( + ` + UPDATE powersync_migration_locks + SET lock_id = ?, expires_at = ? + WHERE name = ? + AND (lock_id IS NULL OR expires_at <= ?) + `, + [options.lockId, options.expiresAt.getTime(), options.name, options.now.getTime()] + ); + + return result.affectedRows == 1; + } + + protected async refreshLock(lockId: string): Promise { + const result = await this.execute( + ` + UPDATE powersync_migration_locks + SET expires_at = ? + WHERE name = ? AND lock_id = ? + `, + [new Date(Date.now() + this.timeout).getTime(), this.name, lockId] + ); + + if (result.affectedRows != 1) { + throw new Error('Lock not found, could not refresh'); + } + } + + protected async releaseLock(lockId: string): Promise { + const result = await this.execute( + ` + UPDATE powersync_migration_locks + SET lock_id = NULL, expires_at = 0 + WHERE name = ? AND lock_id = ? + `, + [this.name, lockId] + ); + + if (result.affectedRows != 1) { + throw new Error('Lock not found, could not release'); + } + } + + /** + * Execute raw SQLite against MikroORM's connection so the lock can exist before entity-backed schema exists. + */ + private async execute(sql: string, params: unknown[]): Promise<{ affectedRows?: number }> { + const orm = await this.getOrm(); + return orm.em.getConnection().execute(sql, params, 'run'); + } +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMikroOrmStorageFactory.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMikroOrmStorageFactory.ts new file mode 100644 index 000000000..d7a845e60 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMikroOrmStorageFactory.ts @@ -0,0 +1,18 @@ +import { MikroORM } from '@mikro-orm/core'; +import { MikroOrmBucketStorageFactory } from '../../storage/MikroOrmBucketStorageFactory.js'; +import { NormalizedMikroOrmSqliteStorageConfig } from '../../types/types.js'; +import { createSqliteMikroOrm } from './sqlite-config.js'; +import { sqliteMikroOrmStorageDialect } from './sqlite-dialect.js'; + +export async function createSqliteMikroOrmStorageFactory(options: { + config: NormalizedMikroOrmSqliteStorageConfig; + slotNamePrefix: string; + orm?: MikroORM; +}): Promise { + const orm = options.orm ?? (await createSqliteMikroOrm(options.config)); + return new MikroOrmBucketStorageFactory({ + orm, + dialect: sqliteMikroOrmStorageDialect, + slotNamePrefix: options.slotNamePrefix + }); +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts new file mode 100644 index 000000000..0d71d2381 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts @@ -0,0 +1,26 @@ +import { Migrator } from '@mikro-orm/migrations'; +import { defineConfig, MikroORM, SqliteDriver } from '@mikro-orm/sqlite'; +import { NormalizedMikroOrmSqliteStorageConfig } from '../../types/types.js'; +import { sqliteMikroOrmStorageDialect } from './sqlite-dialect.js'; + +export const SQLITE_MIKRO_ORM_MIGRATIONS_PATH = new URL('../../migrations/sqlite', import.meta.url).pathname; + +export function createSqliteMikroOrmOptions(config: NormalizedMikroOrmSqliteStorageConfig) { + return defineConfig({ + driver: SqliteDriver, + dbName: config.filename, + entities: sqliteMikroOrmStorageDialect.entityClasses, + extensions: [Migrator], + migrations: { + path: config.migrations_path ?? SQLITE_MIKRO_ORM_MIGRATIONS_PATH, + pathTs: config.migrations_path ?? SQLITE_MIKRO_ORM_MIGRATIONS_PATH, + glob: '!(*.d).{js,ts}', + emit: 'ts', + snapshot: false + } + }); +} + +export async function createSqliteMikroOrm(config: NormalizedMikroOrmSqliteStorageConfig): Promise { + return MikroORM.init(createSqliteMikroOrmOptions(config)); +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts new file mode 100644 index 000000000..394c39de3 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts @@ -0,0 +1,92 @@ +import type { SqlEntityManager } from '@mikro-orm/sql'; +import { + BucketData, + BucketParameters, + CurrentData, + Instance, + SourceTable, + SyncRules, + WriteCheckpoint +} from '../../entities/entities-index.js'; +import { MikroOrmCheckpointWatcher, MikroOrmStorageDialect } from '../../storage/MikroOrmStorageDialect.js'; +import { MIKRO_ORM_SQLITE_STORAGE_TYPE } from '../../types/types.js'; + +export class SqliteCheckpointWatcher implements MikroOrmCheckpointWatcher { + private readonly listeners = new Set<() => void>(); + + notify(): void { + for (const listener of this.listeners) { + listener(); + } + } + + async *watch(signal: AbortSignal): AsyncIterable { + while (!signal.aborted) { + yield await new Promise((resolve) => { + const listener = () => { + this.listeners.delete(listener); + resolve(); + }; + this.listeners.add(listener); + signal.addEventListener( + 'abort', + () => { + this.listeners.delete(listener); + resolve(); + }, + { once: true } + ); + }); + } + } +} + +export const sqliteMikroOrmStorageDialect: MikroOrmStorageDialect = { + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + entityClasses: [BucketData, BucketParameters, CurrentData, Instance, SourceTable, SyncRules, WriteCheckpoint], + bucketDataEntity: BucketData, + bucketParametersEntity: BucketParameters, + currentDataEntity: CurrentData, + instanceEntity: Instance, + sourceTableEntity: SourceTable, + syncRulesEntity: SyncRules, + writeCheckpointEntity: WriteCheckpoint, + async *streamBucketDataRows(options) { + if (options.dataBuckets.length == 0) { + return; + } + + const filters = options.dataBuckets.map((request) => ({ + bucketName: request.bucket, + opId: { $gt: request.start, $lte: options.checkpoint } + })); + const sqlEntityManager = options.em as SqlEntityManager; + const queryBuilder = sqlEntityManager + .createQueryBuilder(BucketData, 'bucket_data') + .select('*') + .where({ + groupId: options.groupId, + $or: filters + }) + .orderBy({ bucketName: 'ASC', opId: 'ASC' }) + .limit(options.limit); + + yield* streamQueryBuilder(queryBuilder.stream()); + }, + createCheckpointWatcher: () => new SqliteCheckpointWatcher() +}; + +async function* streamQueryBuilder(stream: AsyncIterableIterator): AsyncIterable { + const iterator = stream[Symbol.asyncIterator](); + try { + while (true) { + const result = await iterator.next(); + if (result.done) { + return; + } + yield result.value; + } + } finally { + await iterator.return?.(); + } +} diff --git a/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts b/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts new file mode 100644 index 000000000..766018692 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts @@ -0,0 +1,46 @@ +import { defineEntity, p } from '@mikro-orm/core'; + +export class BucketData { + declare id: string; + declare groupId: number; + declare bucketName: string; + declare opId: bigint; + declare op: string; + declare sourceTable: string | null; + declare sourceKey: Buffer | Uint8Array | null; + declare tableName: string | null; + declare rowId: string | null; + declare checksum: bigint; + declare data: string | null; + declare targetOp: bigint | null; +} + +export const BucketDataSchema = defineEntity({ + name: 'BucketData', + tableName: 'bucket_data', + indexes: [ + { + name: 'bucket_data_bucket_op_index', + properties: ['groupId', 'bucketName', 'opId'] + }, + { + name: 'bucket_data_source_index', + properties: ['groupId', 'sourceTable', 'sourceKey'] + } + ], + properties: { + id: p.string().primary(), + groupId: p.integer().fieldName('group_id'), + bucketName: p.string().fieldName('bucket_name'), + opId: p.bigint('bigint'), + op: p.string(), + sourceTable: p.string().fieldName('source_table').nullable(), + sourceKey: p.blob().nullable(), + tableName: p.string().fieldName('table_name').nullable(), + rowId: p.string().fieldName('row_id').nullable(), + checksum: p.bigint('bigint'), + data: p.string().nullable(), + targetOp: p.bigint('bigint').nullable() + } +}); +BucketDataSchema.setClass(BucketData); diff --git a/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts b/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts new file mode 100644 index 000000000..10f144d13 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts @@ -0,0 +1,34 @@ +import { defineEntity, p } from '@mikro-orm/core'; + +export class BucketParameters { + declare id: bigint; + declare groupId: number; + declare sourceTable: string; + declare sourceKey: Buffer | Uint8Array; + declare lookup: Buffer | Uint8Array; + declare bucketParameters: unknown; +} + +export const BucketParametersSchema = defineEntity({ + name: 'BucketParameters', + tableName: 'bucket_parameters', + indexes: [ + { + name: 'bucket_parameters_lookup_index', + properties: ['groupId', 'lookup', 'id'] + }, + { + name: 'bucket_parameters_source_index', + properties: ['groupId', 'sourceTable', 'sourceKey'] + } + ], + properties: { + id: p.bigint('bigint').primary().autoincrement(false), + groupId: p.integer().fieldName('group_id'), + sourceTable: p.string().fieldName('source_table'), + sourceKey: p.blob(), + lookup: p.blob(), + bucketParameters: p.json() + } +}); +BucketParametersSchema.setClass(BucketParameters); diff --git a/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts b/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts new file mode 100644 index 000000000..090e14ca3 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts @@ -0,0 +1,38 @@ +import { defineEntity, p } from '@mikro-orm/core'; + +export class CurrentData { + declare id: string; + declare groupId: number; + declare sourceTable: string; + declare sourceKey: Buffer | Uint8Array; + declare buckets: unknown; + declare lookups: unknown; + declare data: Buffer | Uint8Array; + declare pendingDelete: bigint | null; +} + +export const CurrentDataSchema = defineEntity({ + name: 'CurrentData', + tableName: 'current_data', + indexes: [ + { + name: 'current_data_source_index', + properties: ['groupId', 'sourceTable', 'sourceKey'] + }, + { + name: 'current_data_pending_delete_index', + properties: ['groupId', 'pendingDelete'] + } + ], + properties: { + id: p.string().primary(), + groupId: p.integer().fieldName('group_id'), + sourceTable: p.string().fieldName('source_table'), + sourceKey: p.blob(), + buckets: p.json(), + lookups: p.json(), + data: p.blob(), + pendingDelete: p.bigint('bigint').nullable() + } +}); +CurrentDataSchema.setClass(CurrentData); diff --git a/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts b/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts new file mode 100644 index 000000000..c38d2d0bc --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts @@ -0,0 +1,14 @@ +import { defineEntity, p } from '@mikro-orm/core'; + +export class Instance { + declare id: string; +} + +export const InstanceSchema = defineEntity({ + name: 'Instance', + tableName: 'instance', + properties: { + id: p.string().primary() + } +}); +InstanceSchema.setClass(Instance); diff --git a/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts b/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts new file mode 100644 index 000000000..7c78ab398 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts @@ -0,0 +1,41 @@ +import type { Opt } from '@mikro-orm/core'; +import { defineEntity, p } from '@mikro-orm/core'; + +export class SourceTable { + declare id: string; + declare groupId: number; + declare connectionId: number; + declare relationId: unknown | null; + declare schemaName: string; + declare tableName: string; + declare replicaIdColumns: unknown | null; + declare snapshotDone: Opt; + declare snapshotTotalEstimatedCount: bigint | null; + declare snapshotReplicatedCount: bigint | null; + declare snapshotLastKey: Buffer | Uint8Array | null; +} + +export const SourceTableSchema = defineEntity({ + name: 'SourceTable', + tableName: 'source_tables', + indexes: [ + { + name: 'source_table_lookup', + properties: ['groupId', 'tableName'] + } + ], + properties: { + id: p.string().primary(), + groupId: p.integer().fieldName('group_id'), + connectionId: p.integer().fieldName('connection_id'), + relationId: p.json().nullable(), + schemaName: p.string().fieldName('schema_name'), + tableName: p.string().fieldName('table_name'), + replicaIdColumns: p.json().nullable(), + snapshotDone: p.boolean().fieldName('snapshot_done').default(true), + snapshotTotalEstimatedCount: p.bigint('bigint').nullable(), + snapshotReplicatedCount: p.bigint('bigint').nullable(), + snapshotLastKey: p.blob().nullable() + } +}); +SourceTableSchema.setClass(SourceTable); diff --git a/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts b/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts new file mode 100644 index 000000000..de88c7ad8 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts @@ -0,0 +1,46 @@ +import type { Opt } from '@mikro-orm/core'; +import { defineEntity, p } from '@mikro-orm/core'; +import type { storage } from '@powersync/service-core'; + +export class SyncRules { + declare id: Opt; + declare state: storage.SyncRuleState; + declare snapshotDone: Opt; + declare snapshotLsn: string | null; + declare lastCheckpoint: bigint | null; + declare lastCheckpointLsn: string | null; + declare noCheckpointBefore: string | null; + declare slotName: string; + declare lastCheckpointTs: Date | null; + declare lastKeepaliveTs: Date | null; + declare lastFatalError: string | null; + declare lastFatalErrorTs: Date | null; + declare keepaliveOp: bigint | null; + declare storageVersion: number | null; + declare content: string; + declare syncPlan: storage.SerializedSyncPlan | null; +} + +export const SyncRulesSchema = defineEntity({ + name: 'SyncRules', + tableName: 'sync_rules', + properties: { + id: p.integer().primary().autoincrement(), + state: p.string(), + snapshotDone: p.boolean().fieldName('snapshot_done').default(false), + snapshotLsn: p.string().fieldName('snapshot_lsn').nullable(), + lastCheckpoint: p.bigint('bigint').nullable(), + lastCheckpointLsn: p.string().fieldName('last_checkpoint_lsn').nullable(), + noCheckpointBefore: p.string().fieldName('no_checkpoint_before').nullable(), + slotName: p.string().fieldName('slot_name'), + lastCheckpointTs: p.datetime().nullable(), + lastKeepaliveTs: p.datetime().nullable(), + lastFatalError: p.string().fieldName('last_fatal_error').nullable(), + lastFatalErrorTs: p.datetime().nullable(), + keepaliveOp: p.bigint('bigint').nullable(), + storageVersion: p.integer().fieldName('storage_version').nullable(), + content: p.string(), + syncPlan: p.json().nullable() + } +}); +SyncRulesSchema.setClass(SyncRules); diff --git a/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts b/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts new file mode 100644 index 000000000..36b4b5487 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts @@ -0,0 +1,30 @@ +import { defineEntity, p } from '@mikro-orm/core'; + +export class WriteCheckpoint { + declare id: string; + declare syncRulesId: number | null; + declare userId: string; + declare checkpoint: bigint; + declare heads: unknown | null; + declare createdAt: Date; +} + +export const WriteCheckpointSchema = defineEntity({ + name: 'WriteCheckpoint', + tableName: 'write_checkpoints', + indexes: [ + { + name: 'write_checkpoints_user_checkpoint_index', + properties: ['userId', 'syncRulesId', 'checkpoint'] + } + ], + properties: { + id: p.string().primary(), + syncRulesId: p.integer().fieldName('sync_rules_id').nullable(), + userId: p.string().fieldName('user_id'), + checkpoint: p.bigint('bigint'), + heads: p.json().nullable(), + createdAt: p.datetime() + } +}); +WriteCheckpointSchema.setClass(WriteCheckpoint); diff --git a/modules/module-mikroorm-storage/src/entities/entities-index.ts b/modules/module-mikroorm-storage/src/entities/entities-index.ts new file mode 100644 index 000000000..13728f1cb --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/entities-index.ts @@ -0,0 +1,7 @@ +export * from './common/bucket-data.schema.js'; +export * from './common/bucket-parameters.schema.js'; +export * from './common/current-data.schema.js'; +export * from './common/instance.schema.js'; +export * from './common/source-table.schema.js'; +export * from './common/sync-rules.schema.js'; +export * from './common/write-checkpoint.schema.js'; diff --git a/modules/module-mikroorm-storage/src/index.ts b/modules/module-mikroorm-storage/src/index.ts new file mode 100644 index 000000000..28b08fde4 --- /dev/null +++ b/modules/module-mikroorm-storage/src/index.ts @@ -0,0 +1,10 @@ +export * from './drivers/sqlite/sqlite-config.js'; +export * from './drivers/sqlite/sqlite-dialect.js'; +export * from './drivers/sqlite/SqliteMikroOrmStorageFactory.js'; +export * from './entities/entities-index.js'; +export * as entities from './entities/entities-index.js'; +export * from './migrations/MikroOrmMigrationAgent.js'; +export * from './module/MikroOrmStorageModule.js'; +export * from './storage/storage-index.js'; +export * as storage from './storage/storage-index.js'; +export * from './types/types.js'; diff --git a/modules/module-mikroorm-storage/src/migrations/AbstractMikroOrmMigrationLockManager.ts b/modules/module-mikroorm-storage/src/migrations/AbstractMikroOrmMigrationLockManager.ts new file mode 100644 index 000000000..89a76b7d2 --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/AbstractMikroOrmMigrationLockManager.ts @@ -0,0 +1,101 @@ +import { MikroORM } from '@mikro-orm/core'; +import { locks } from '@powersync/lib-services-framework'; +import * as uuid from 'uuid'; + +const DEFAULT_LOCK_TIMEOUT = 60_000; + +/** + * Construction parameters for a database-backed MikroORM migration lock manager. + * + * The ORM instance is supplied so dialect implementations can reuse MikroORM's configured connection while still + * issuing the small amount of raw SQL required to bootstrap the lock table before migrations have run. + */ +export interface MikroOrmMigrationLockManagerParams extends locks.LockManagerParams { + orm: MikroORM | Promise; +} + +/** + * Base lock manager for MikroORM-backed storage migrations. + * + * The common layer owns lock lifecycle behavior and expiry handling. Dialects provide the actual persistence + * operations because lock bootstrap is database-specific and must work before the normal MikroORM migration schema + * exists. + */ +export abstract class AbstractMikroOrmMigrationLockManager extends locks.AbstractLockManager { + private ormInstance: MikroORM | undefined; + + constructor(protected params: MikroOrmMigrationLockManagerParams) { + super(params); + } + + protected get timeout() { + return this.params.timeout ?? DEFAULT_LOCK_TIMEOUT; + } + + protected get name() { + return this.params.name; + } + + protected async getOrm(): Promise { + this.ormInstance ??= await this.params.orm; + return this.ormInstance; + } + + async init(): Promise { + await this.initLockStore(); + } + + protected async acquireHandle(): Promise { + const lockId = await this.acquireLockId(); + if (lockId == null) { + return null; + } + + return { + refresh: () => this.refreshLock(lockId), + release: () => this.releaseLock(lockId) + }; + } + + protected async acquireLockId(): Promise { + const lockId = uuid.v4(); + const now = new Date(); + const expiresAt = new Date(now.getTime() + this.timeout); + + return (await this.tryAcquireLock({ + name: this.name, + lockId, + now, + expiresAt + })) + ? lockId + : null; + } + + /** + * Create or prepare the dialect-specific lock storage. + * + * Implementations must be safe to call before any generated MikroORM migration has run. + */ + protected abstract initLockStore(): Promise; + + /** + * Try to acquire the named lock if it is free or expired. + */ + protected abstract tryAcquireLock(options: { + name: string; + lockId: string; + now: Date; + expiresAt: Date; + }): Promise; + + /** + * Extend ownership of the currently-held lock. + */ + protected abstract refreshLock(lockId: string): Promise; + + /** + * Release ownership of the currently-held lock. + */ + protected abstract releaseLock(lockId: string): Promise; +} diff --git a/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts b/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts new file mode 100644 index 000000000..b05cf786d --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts @@ -0,0 +1,93 @@ +import { MikroORM } from '@mikro-orm/core'; +import * as framework from '@powersync/lib-services-framework'; +import { migrations } from '@powersync/service-core'; +import { createSqliteMikroOrm } from '../drivers/sqlite/sqlite-config.js'; +import { SqliteMigrationLockManager } from '../drivers/sqlite/SqliteMigrationLockManager.js'; +import { MikroOrmSqliteStorageConfigDecoded, normalizeMikroOrmSqliteStorageConfig } from '../types/types.js'; +import { NoOpMigrationStore } from './NoOpMigrationStore.js'; + +/** + * Service migration agent that delegates schema changes to MikroORM migrations. + * + * PowerSync still owns orchestration and locking, but MikroORM owns migration discovery and migration state. The + * lock is acquired first to avoid multiple service instances running the same MikroORM migration concurrently. + */ +export class MikroOrmMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent { + store: framework.MigrationStore; + locks: framework.LockManager; + + private readonly ormPromise: Promise; + + constructor(config: MikroOrmSqliteStorageConfigDecoded) { + super(); + this.ormPromise = createSqliteMikroOrm(normalizeMikroOrmSqliteStorageConfig(config)); + this.store = new NoOpMigrationStore(); + this.locks = new SqliteMigrationLockManager({ + name: 'mikroorm-migrations', + orm: this.ormPromise + }); + } + + getInternalScriptsDir(): string { + return new URL('./scripts', import.meta.url).pathname; + } + + async run(params: framework.RunMigrationParams): Promise { + await this.locks.init?.(); + + const logger = params.logger ?? framework.logger; + logger.debug('Acquiring lock for MikroORM migrations'); + const lockHandle = await this.locks.acquire({ + max_wait_ms: params.maxLockWaitMs ?? framework.DEFAULT_MAX_LOCK_WAIT_MS + }); + + if (lockHandle == null) { + throw new Error('Could not acquire MikroORM migration lock'); + } + + let isReleased = false; + const releaseLock = async () => { + if (isReleased) { + return; + } + await lockHandle.release(); + isReleased = true; + }; + + process.addListener('beforeExit', releaseLock); + + try { + if (params.migrations.length > 0) { + logger.warn('Ignoring PowerSync migration list for MikroORM storage; MikroORM migrator owns migration state.'); + } + + const orm = await this.ormPromise; + const migrator = orm.migrator; + + logger.info(`Running MikroORM migrations ${params.direction}`); + if (migrator != null) { + if (params.direction == framework.Direction.Up) { + await migrator.up(); + } else { + await migrator.down(); + } + } else if (params.direction == framework.Direction.Up) { + await orm.schema.update(); + } + } finally { + logger.debug('Releasing MikroORM migration lock'); + await releaseLock(); + process.removeListener('beforeExit', releaseLock); + logger.debug('Done with MikroORM migrations'); + } + } + + async loadInternalMigrations(): Promise[]> { + return []; + } + + async [Symbol.asyncDispose](): Promise { + const orm = await this.ormPromise; + await orm.close(true); + } +} diff --git a/modules/module-mikroorm-storage/src/migrations/NoOpMigrationStore.ts b/modules/module-mikroorm-storage/src/migrations/NoOpMigrationStore.ts new file mode 100644 index 000000000..2f3c5b08d --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/NoOpMigrationStore.ts @@ -0,0 +1,15 @@ +import { migrations } from '@powersync/lib-services-framework'; + +/** + * Service migrations only trigger MikroORM's migrator for this module. + * MikroORM owns the actual migration state in its own migration storage. + */ +export class NoOpMigrationStore implements migrations.MigrationStore { + async load(): Promise { + return undefined; + } + + async save(_state: migrations.MigrationState): Promise {} + + async clear(): Promise {} +} diff --git a/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612083946_InitialSqliteStorage.ts b/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612083946_InitialSqliteStorage.ts new file mode 100644 index 000000000..236e92580 --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612083946_InitialSqliteStorage.ts @@ -0,0 +1,53 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20260612083946_InitialSqliteStorage extends Migration { + override async up(): Promise { + this.addSql( + `create table \`bucket_data\` (\`id\` text not null, \`group_id\` integer not null, \`bucket_name\` text not null, \`op_id\` bigint not null, \`op\` text not null, \`source_table\` text null, \`source_key\` blob null, \`table_name\` text null, \`row_id\` text null, \`checksum\` bigint not null, \`data\` text null, \`target_op\` bigint null, primary key (\`id\`));` + ); + this.addSql( + `create index \`bucket_data_bucket_op_index\` on \`bucket_data\` (\`group_id\`, \`bucket_name\`, \`op_id\`);` + ); + this.addSql( + `create index \`bucket_data_source_index\` on \`bucket_data\` (\`group_id\`, \`source_table\`, \`source_key\`);` + ); + + this.addSql( + `create table \`bucket_parameters\` (\`id\` bigint not null, \`group_id\` integer not null, \`source_table\` text not null, \`source_key\` blob not null, \`lookup\` blob not null, \`bucket_parameters\` json not null, primary key (\`id\`));` + ); + this.addSql( + `create index \`bucket_parameters_lookup_index\` on \`bucket_parameters\` (\`group_id\`, \`lookup\`, \`id\`);` + ); + this.addSql( + `create index \`bucket_parameters_source_index\` on \`bucket_parameters\` (\`group_id\`, \`source_table\`, \`source_key\`);` + ); + + this.addSql( + `create table \`current_data\` (\`id\` text not null, \`group_id\` integer not null, \`source_table\` text not null, \`source_key\` blob not null, \`buckets\` json not null, \`lookups\` json not null, \`data\` blob not null, \`pending_delete\` bigint null, primary key (\`id\`));` + ); + this.addSql( + `create index \`current_data_source_index\` on \`current_data\` (\`group_id\`, \`source_table\`, \`source_key\`);` + ); + this.addSql( + `create index \`current_data_pending_delete_index\` on \`current_data\` (\`group_id\`, \`pending_delete\`);` + ); + + this.addSql(`create table \`instance\` (\`id\` text not null, primary key (\`id\`));`); + + this.addSql( + `create table \`source_tables\` (\`id\` text not null, \`group_id\` integer not null, \`connection_id\` integer not null, \`relation_id\` json null, \`schema_name\` text not null, \`table_name\` text not null, \`replica_id_columns\` json null, \`snapshot_done\` integer not null default true, \`snapshot_total_estimated_count\` bigint null, \`snapshot_replicated_count\` bigint null, \`snapshot_last_key\` blob null, primary key (\`id\`));` + ); + this.addSql(`create index \`source_table_lookup\` on \`source_tables\` (\`group_id\`, \`table_name\`);`); + + this.addSql( + `create table \`sync_rules\` (\`id\` integer not null primary key autoincrement, \`state\` text not null, \`snapshot_done\` integer not null default false, \`snapshot_lsn\` text null, \`last_checkpoint\` bigint null, \`last_checkpoint_lsn\` text null, \`no_checkpoint_before\` text null, \`slot_name\` text not null, \`last_checkpoint_ts\` datetime null, \`last_keepalive_ts\` datetime null, \`last_fatal_error\` text null, \`last_fatal_error_ts\` datetime null, \`keepalive_op\` bigint null, \`storage_version\` integer null, \`content\` text not null, \`sync_plan\` json null);` + ); + + this.addSql( + `create table \`write_checkpoints\` (\`id\` text not null, \`sync_rules_id\` integer null, \`user_id\` text not null, \`checkpoint\` bigint not null, \`heads\` json null, \`created_at\` datetime not null, primary key (\`id\`));` + ); + this.addSql( + `create index \`write_checkpoints_user_checkpoint_index\` on \`write_checkpoints\` (\`user_id\`, \`sync_rules_id\`, \`checkpoint\`);` + ); + } +} diff --git a/modules/module-mikroorm-storage/src/mikro-orm.config.ts b/modules/module-mikroorm-storage/src/mikro-orm.config.ts new file mode 100644 index 000000000..86f7261e1 --- /dev/null +++ b/modules/module-mikroorm-storage/src/mikro-orm.config.ts @@ -0,0 +1,43 @@ +import type { Options } from '@mikro-orm/sqlite'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createSqliteMikroOrmOptions } from './drivers/sqlite/sqlite-config.js'; +import { MIKRO_ORM_SQLITE_STORAGE_TYPE, normalizeMikroOrmSqliteStorageConfig } from './types/types.js'; + +type MikroOrmStorageDialectName = 'sqlite'; + +const moduleRoot = fileURLToPath(new URL('..', import.meta.url)); + +function getDialect(): MikroOrmStorageDialectName { + const dialect = process.env.MIKRO_ORM_STORAGE_DIALECT ?? 'sqlite'; + if (dialect == 'sqlite') { + return dialect; + } + + throw new Error(`Unsupported MikroORM storage dialect for migration generation: ${dialect}`); +} + +function sqliteConfig(): Options { + const options = createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: process.env.MIKRO_ORM_SQLITE_DB ?? ':memory:' + }) + ); + + return { + ...options, + migrations: { + ...options.migrations, + path: resolve(moduleRoot, 'dist/migrations/sqlite'), + pathTs: resolve(moduleRoot, 'src/migrations/sqlite'), + emit: 'ts' + } + }; +} + +const configs = { + sqlite: sqliteConfig +} satisfies Record Options>; + +export default configs[getDialect()](); diff --git a/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts b/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts new file mode 100644 index 000000000..f9f131e97 --- /dev/null +++ b/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts @@ -0,0 +1,26 @@ +import { modules, system } from '@powersync/service-core'; +import { MikroOrmMigrationAgent } from '../migrations/MikroOrmMigrationAgent.js'; +import { MikroOrmStorageProvider } from '../storage/MikroOrmStorageProvider.js'; +import { isMikroOrmSqliteStorageConfig, MikroOrmSqliteStorageConfig } from '../types/types.js'; + +export class MikroOrmStorageModule extends modules.AbstractModule { + constructor() { + super({ + name: 'MikroORM Bucket Storage' + }); + } + + async initialize(context: system.ServiceContextContainer): Promise { + context.storageEngine.registerProvider(new MikroOrmStorageProvider()); + + if (isMikroOrmSqliteStorageConfig(context.configuration.storage)) { + context.migrations.registerMigrationAgent( + new MikroOrmMigrationAgent(MikroOrmSqliteStorageConfig.decode(context.configuration.storage)) + ); + } + } + + async teardown(): Promise { + // Teardown for this module is implemented by the storage engine. + } +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts new file mode 100644 index 000000000..b39992a6d --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts @@ -0,0 +1,637 @@ +import { EntityManager, MikroORM } from '@mikro-orm/core'; +import { BaseObserver, DO_NOT_LOG, Logger, ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { ColumnDescriptor, storage, utils } from '@powersync/service-core'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as uuid from 'uuid'; +import { SourceTable as SourceTableEntity } from '../entities/entities-index.js'; +import { MikroOrmBucketStorageFactory } from './MikroOrmBucketStorageFactory.js'; +import { currentBuckets, currentLookups, MikroOrmPersistedBatch } from './MikroOrmPersistedBatch.js'; +import { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +export interface MikroOrmBucketBatchOptions { + factory: MikroOrmBucketStorageFactory; + orm: MikroORM; + dialect: MikroOrmStorageDialect; + logger: Logger; + syncRules: sync_rules.HydratedSyncConfig; + replicationStreamId: number; + replicationStreamName: string; + lastCheckpointLsn: string | null; + keepaliveOp: bigint | null; + resumeFromLsn: string | null; + storeCurrentData: boolean; + skipExistingRows: boolean; + markRecordUnavailable: storage.BucketStorageMarkRecordUnavailable | undefined; + hooks: storage.StorageHooks | undefined; +} + +const MAX_OPERATION_BATCH_COUNT = 2_000; + +export class MikroOrmBucketBatch + extends BaseObserver + implements storage.BucketStorageBatch +{ + [DO_NOT_LOG] = true; + + public last_flushed_op: bigint | null = null; + public resumeFromLsn: string | null; + public readonly skipExistingRows: boolean; + + private lastCheckpointLsnValue: string | null; + private persistedOp: bigint | null; + private readonly pendingOperations: storage.SaveOptions[] = []; + private readonly customWriteCheckpointBatch: storage.CustomWriteCheckpointOptions[] = []; + private needsActivation = true; + + constructor(private readonly options: MikroOrmBucketBatchOptions) { + super(); + this.lastCheckpointLsnValue = options.lastCheckpointLsn; + this.resumeFromLsn = options.resumeFromLsn; + this.skipExistingRows = options.skipExistingRows; + this.persistedOp = options.keepaliveOp; + } + + get lastCheckpointLsn(): string | null { + return this.lastCheckpointLsnValue; + } + + async [Symbol.asyncDispose](): Promise { + if (this.customWriteCheckpointBatch.length > 0) { + this.options.logger.warn('Disposing writer with unflushed custom write checkpoints'); + } + super.clearListeners(); + } + + async dispose(): Promise { + await this[Symbol.asyncDispose](); + } + + async resolveTables(options: storage.ResolveTablesOptions): Promise { + const syncRules = options.syncRules ?? this.options.syncRules; + const { connection_id, source } = options; + const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source; + const normalizedReplicaIdColumns = normalizeReplicaIdColumns(replicaIdColumns); + const relationId = { object_id: objectId }; + const em = this.options.orm.em.fork(); + + return em.transactional(async (transactionalEntityManager) => { + const existingRows = await transactionalEntityManager.find(this.options.dialect.sourceTableEntity, { + groupId: this.options.replicationStreamId, + connectionId: connection_id, + schemaName: schema, + tableName: table + }); + + let sourceTableRow = + existingRows.find((row) => { + const matchesRelationId = objectId == null || relationObjectId(row.relationId) == objectId; + return matchesRelationId && jsonEquals(row.replicaIdColumns, normalizedReplicaIdColumns); + }) ?? null; + + if (sourceTableRow == null) { + sourceTableRow = transactionalEntityManager.create(this.options.dialect.sourceTableEntity, { + id: options.idGenerator ? String(options.idGenerator()) : uuid.v4(), + groupId: this.options.replicationStreamId, + connectionId: connection_id, + relationId, + schemaName: schema, + tableName: table, + replicaIdColumns: normalizedReplicaIdColumns, + snapshotDone: true, + snapshotTotalEstimatedCount: null, + snapshotReplicatedCount: null, + snapshotLastKey: null + }); + transactionalEntityManager.persist(sourceTableRow); + await transactionalEntityManager.flush(); + } + + const sourceTable = sourceTableFromRow(sourceTableRow, connectionTag, syncRules); + sourceTable.storeCurrentData = sendsCompleteRows !== true; + + const dropTables = existingRows + .filter((row) => row.id != sourceTableRow.id) + .filter((row) => objectId == null || relationObjectId(row.relationId) == objectId) + .map((row) => sourceTableFromRow(row, connectionTag, syncRules)); + + return { + tables: [sourceTable], + dropTables + }; + }); + } + + async getSourceTableStatus(table: storage.SourceTable): Promise { + const row = await this.options.orm.em.fork().findOne(this.options.dialect.sourceTableEntity, { + groupId: this.options.replicationStreamId, + id: String(table.id) + }); + + return row == null ? null : sourceTableFromRow(row, table.ref.connectionTag, this.options.syncRules); + } + + async save(record: storage.SaveOptions): Promise { + const { after, before, sourceTable, tag } = record; + const storeCurrentData = this.options.storeCurrentData && sourceTable.storeCurrentData; + for (const event of this.getTableEvents(sourceTable)) { + this.iterateListeners((cb) => + cb.replicationEvent?.({ + batch: this, + table: sourceTable, + data: { + op: tag, + after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined, + before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined + }, + event + }) + ); + } + + if (!sourceTable.syncData && !sourceTable.syncParameters) { + return null; + } + + this.pendingOperations.push(record); + if (this.pendingOperations.length >= MAX_OPERATION_BATCH_COUNT) { + return this.flush(); + } + return null; + } + + async truncate(sourceTables: storage.SourceTable[]): Promise { + await this.flush(); + + const em = this.options.orm.em.fork(); + let nextOpId = await this.getNextOpId(em); + const firstOpId = nextOpId; + + await em.transactional(async (transactionalEntityManager) => { + const persistedBatch = this.createPersistedBatch(transactionalEntityManager); + for (const table of sourceTables) { + if (!table.syncData && !table.syncParameters) { + continue; + } + + const rows = await transactionalEntityManager.find(this.options.dialect.currentDataEntity, { + groupId: this.options.replicationStreamId, + sourceTable: String(table.id) + }); + + for (const row of rows) { + const sourceKey = storage.deserializeReplicaId(Buffer.from(row.sourceKey)); + if (table.syncData) { + nextOpId = persistedBatch.persistBucketData({ + table, + sourceKey, + existingBuckets: currentBuckets(row), + evaluated: [], + nextOpId + }); + } + + if (table.syncParameters) { + nextOpId = persistedBatch.persistParameterData({ + table, + sourceKey, + existingLookups: currentLookups(row), + evaluated: [], + nextOpId + }); + } + + transactionalEntityManager.remove(row); + } + } + await transactionalEntityManager.flush(); + }); + + if (nextOpId == firstOpId) { + return null; + } + + const lastOpId = nextOpId - 1n; + this.persistedOp = lastOpId; + this.last_flushed_op = lastOpId; + return { flushed_op: lastOpId }; + } + + async drop(sourceTables: storage.SourceTable[]): Promise { + const result = await this.truncate(sourceTables); + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + for (const table of sourceTables) { + await transactionalEntityManager.nativeDelete(this.options.dialect.sourceTableEntity, { + groupId: this.options.replicationStreamId, + id: String(table.id) + }); + } + }); + return result; + } + + async flush(_options?: storage.BatchBucketFlushOptions): Promise { + let result: storage.FlushedResult | null = null; + if (this.pendingOperations.length > 0) { + await this.options.hooks?.beforeBatchFlush?.(this); + const operations = this.pendingOperations.splice(0); + const em = this.options.orm.em.fork(); + let nextOpId = await this.getNextOpId(em); + for (const batch of chunked(operations, MAX_OPERATION_BATCH_COUNT)) { + await this.options.orm.em.fork().transactional(async (transactionalEntityManager) => { + const persistedBatch = this.createPersistedBatch(transactionalEntityManager); + nextOpId = await persistedBatch.persistOperations(batch, nextOpId); + await transactionalEntityManager.flush(); + }); + } + const lastOpId = nextOpId - 1n; + this.persistedOp = lastOpId; + this.last_flushed_op = lastOpId; + result = { flushed_op: lastOpId }; + await this.options.hooks?.afterBatchFlush?.(this); + } + + await this.flushCustomWriteCheckpoints(); + return result; + } + + async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { + await this.flush(); + const createEmptyCheckpoints = options?.createEmptyCheckpoints ?? true; + const now = new Date(); + const em = this.options.orm.em.fork(); + + const result = await em.transactional(async (transactionalEntityManager) => { + const syncRulesRow = await transactionalEntityManager.findOneOrFail(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + + const canCheckpoint = + syncRulesRow.snapshotDone === true && + (syncRulesRow.lastCheckpointLsn == null || syncRulesRow.lastCheckpointLsn <= lsn) && + (syncRulesRow.noCheckpointBefore == null || syncRulesRow.noCheckpointBefore <= lsn); + + let checkpointCreated = false; + + if (canCheckpoint) { + const newLastCheckpoint = maxBigint( + syncRulesRow.lastCheckpoint, + this.persistedOp, + syncRulesRow.keepaliveOp, + 0n + ); + const changed = syncRulesRow.lastCheckpoint !== newLastCheckpoint || syncRulesRow.keepaliveOp != null; + + if (changed || createEmptyCheckpoints) { + transactionalEntityManager.assign(syncRulesRow, { + lastCheckpointLsn: lsn, + lastCheckpointTs: now, + lastKeepaliveTs: now, + lastFatalError: null, + keepaliveOp: null, + lastCheckpoint: newLastCheckpoint, + snapshotLsn: null + }); + checkpointCreated = true; + if (newLastCheckpoint != null) { + await transactionalEntityManager.nativeDelete(this.options.dialect.currentDataEntity, { + groupId: this.options.replicationStreamId, + pendingDelete: { $lte: newLastCheckpoint } + }); + } + } else { + transactionalEntityManager.assign(syncRulesRow, { + lastKeepaliveTs: now + }); + } + } else { + transactionalEntityManager.assign(syncRulesRow, { + keepaliveOp: maxBigint(syncRulesRow.keepaliveOp, this.persistedOp, 0n), + lastKeepaliveTs: now + }); + } + + await transactionalEntityManager.flush(); + return { + checkpointBlocked: !canCheckpoint, + checkpointCreated + }; + }); + + if (!result.checkpointBlocked) { + await this.autoActivate(lsn); + } + + this.persistedOp = null; + this.lastCheckpointLsnValue = lsn; + this.options.factory.checkpointWatcher.notify(); + return result; + } + + keepalive(lsn: string): Promise { + return this.commit(lsn, { createEmptyCheckpoints: true }); + } + + async setResumeLsn(lsn: string): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOneOrFail(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + em.assign(row, { snapshotLsn: lsn }); + await em.flush(); + this.resumeFromLsn = lsn; + } + + async markAllSnapshotDone(noCheckpointBeforeLsn: string): Promise { + await this.markSnapshotDoneInternal(noCheckpointBeforeLsn); + } + + async markSnapshotDone(noCheckpointBeforeLsn: string, options?: { throwOnConflict?: boolean }): Promise { + const remaining = await this.options.orm.em.fork().count(this.options.dialect.sourceTableEntity, { + groupId: this.options.replicationStreamId, + snapshotDone: false + }); + + if (remaining > 0) { + if (options?.throwOnConflict ?? true) { + throw new ReplicationAssertionError( + `Cannot mark snapshot done while ${remaining} source table${remaining == 1 ? '' : 's'} still require snapshotting` + ); + } + return; + } + + await this.markSnapshotDoneInternal(noCheckpointBeforeLsn); + } + + async markTableSnapshotRequired(_table: storage.SourceTable): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOneOrFail(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + em.assign(row, { snapshotDone: false }); + await em.flush(); + } + + async markTableSnapshotDone( + tables: storage.SourceTable[], + noCheckpointBeforeLsn?: string + ): Promise { + const ids = tables.map((table) => String(table.id)); + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + const rows = await transactionalEntityManager.find(this.options.dialect.sourceTableEntity, { + id: { $in: ids } + }); + for (const row of rows) { + transactionalEntityManager.assign(row, { + snapshotDone: true, + snapshotTotalEstimatedCount: null, + snapshotReplicatedCount: null, + snapshotLastKey: null + }); + } + + if (noCheckpointBeforeLsn != null) { + await this.assignNoCheckpointBefore(transactionalEntityManager, noCheckpointBeforeLsn); + } + }); + + return tables.map((table) => { + const copy = table.clone(); + copy.snapshotComplete = true; + copy.snapshotStatus = undefined; + return copy; + }); + } + + async updateTableProgress( + table: storage.SourceTable, + progress: Partial + ): Promise { + const copy = table.clone(); + const snapshotStatus = { + totalEstimatedCount: progress.totalEstimatedCount ?? copy.snapshotStatus?.totalEstimatedCount ?? 0, + replicatedCount: progress.replicatedCount ?? copy.snapshotStatus?.replicatedCount ?? 0, + lastKey: progress.lastKey ?? copy.snapshotStatus?.lastKey ?? null + }; + copy.snapshotStatus = snapshotStatus; + + const em = this.options.orm.em.fork(); + const row = await em.findOneOrFail(this.options.dialect.sourceTableEntity, { + id: String(table.id) + }); + em.assign(row, { + snapshotTotalEstimatedCount: BigInt(snapshotStatus.totalEstimatedCount), + snapshotReplicatedCount: BigInt(snapshotStatus.replicatedCount), + snapshotLastKey: snapshotStatus.lastKey + }); + await em.flush(); + + return copy; + } + + addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void { + this.customWriteCheckpointBatch.push({ + ...checkpoint, + sync_rules_id: this.options.replicationStreamId + }); + } + + private async flushCustomWriteCheckpoints(): Promise { + if (this.customWriteCheckpointBatch.length == 0) { + return; + } + + const batch = this.customWriteCheckpointBatch.splice(0); + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + for (const checkpoint of batch) { + const existing = await transactionalEntityManager.findOne(this.options.dialect.writeCheckpointEntity, { + userId: checkpoint.user_id, + syncRulesId: checkpoint.sync_rules_id + }); + + if (existing == null) { + const row = transactionalEntityManager.create(this.options.dialect.writeCheckpointEntity, { + id: uuid.v4(), + syncRulesId: checkpoint.sync_rules_id, + userId: checkpoint.user_id, + checkpoint: checkpoint.checkpoint, + heads: null, + createdAt: new Date() + }); + transactionalEntityManager.persist(row); + } else { + transactionalEntityManager.assign(existing, { + checkpoint: checkpoint.checkpoint, + createdAt: new Date() + }); + } + } + await transactionalEntityManager.flush(); + }); + this.options.factory.checkpointWatcher.notify(); + } + + private createPersistedBatch(transactionalEntityManager: EntityManager): MikroOrmPersistedBatch { + return new MikroOrmPersistedBatch({ + transactionalEntityManager, + dialect: this.options.dialect, + logger: this.options.logger, + syncRules: this.options.syncRules, + replicationStreamId: this.options.replicationStreamId, + storeCurrentData: this.options.storeCurrentData, + skipExistingRows: this.options.skipExistingRows, + markRecordUnavailable: this.options.markRecordUnavailable + }); + } + + private async getNextOpId(em: EntityManager): Promise { + const [bucketData] = await em.find( + this.options.dialect.bucketDataEntity, + {}, + { orderBy: { opId: 'DESC' }, limit: 1 } + ); + const [bucketParameters] = await em.find( + this.options.dialect.bucketParametersEntity, + {}, + { orderBy: { id: 'DESC' }, limit: 1 } + ); + const [pendingDelete] = await em.find( + this.options.dialect.currentDataEntity, + { pendingDelete: { $ne: null } }, + { orderBy: { pendingDelete: 'DESC' }, limit: 1 } + ); + + return maxBigint(bucketData?.opId, bucketParameters?.id, pendingDelete?.pendingDelete, 0n) + 1n; + } + + private async markSnapshotDoneInternal(noCheckpointBeforeLsn: string): Promise { + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + await this.assignNoCheckpointBefore(transactionalEntityManager, noCheckpointBeforeLsn); + }); + this.options.factory.checkpointWatcher.notify(); + } + + private async assignNoCheckpointBefore( + transactionalEntityManager: EntityManager, + noCheckpointBeforeLsn: string + ): Promise { + const row = await transactionalEntityManager.findOneOrFail(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + transactionalEntityManager.assign(row, { + snapshotDone: true, + lastKeepaliveTs: new Date(), + noCheckpointBefore: + row.noCheckpointBefore == null || row.noCheckpointBefore < noCheckpointBeforeLsn + ? noCheckpointBeforeLsn + : row.noCheckpointBefore + }); + } + + private async autoActivate(lsn: string): Promise { + if (!this.needsActivation) { + return; + } + + const em = this.options.orm.em.fork(); + let didActivate = false; + await em.transactional(async (transactionalEntityManager) => { + const syncRulesRow = await transactionalEntityManager.findOne(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + + if (syncRulesRow?.state == storage.SyncRuleState.PROCESSING && syncRulesRow.snapshotDone) { + transactionalEntityManager.assign(syncRulesRow, { state: storage.SyncRuleState.ACTIVE }); + const oldRows = await transactionalEntityManager.find(this.options.dialect.syncRulesEntity, { + id: { $ne: this.options.replicationStreamId }, + state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] } + }); + for (const oldRow of oldRows) { + transactionalEntityManager.assign(oldRow, { state: storage.SyncRuleState.STOP }); + } + await transactionalEntityManager.flush(); + didActivate = true; + this.needsActivation = false; + } else if (syncRulesRow?.state != storage.SyncRuleState.PROCESSING) { + this.needsActivation = false; + } + }); + + if (didActivate) { + this.options.logger.info(`Activated new replication stream at ${lsn}`); + } + } + + private getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] { + return this.options.syncRules.eventDescriptors.filter((event) => + [...event.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref)) + ); + } +} + +function sourceTableFromRow( + row: SourceTableEntity, + connectionTag: string, + syncRules: sync_rules.HydratedSyncConfig +): storage.SourceTable { + const ref = { connectionTag, schema: row.schemaName, name: row.tableName }; + const sourceTable = new storage.SourceTable({ + id: row.id, + ref, + objectId: relationObjectId(row.relationId), + replicaIdColumns: replicaIdColumns(row.replicaIdColumns), + snapshotComplete: row.snapshotDone ?? true, + ...syncRules.getMatchingSources(ref) + }); + + if (!sourceTable.snapshotComplete) { + sourceTable.snapshotStatus = { + totalEstimatedCount: Number(row.snapshotTotalEstimatedCount ?? -1n), + replicatedCount: Number(row.snapshotReplicatedCount ?? 0n), + lastKey: row.snapshotLastKey ?? null + }; + } + + sourceTable.syncEvent = syncRules.tableTriggersEvent(ref); + sourceTable.syncData = sourceTable.bucketDataSources.length > 0; + sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0; + return sourceTable; +} + +function normalizeReplicaIdColumns(replicaIdColumns: ColumnDescriptor[]): ColumnDescriptor[] { + return replicaIdColumns.map((column) => ({ + name: column.name, + type: column.type, + typeId: column.typeId + })); +} + +function replicaIdColumns(value: unknown): ColumnDescriptor[] { + return Array.isArray(value) ? (value as ColumnDescriptor[]) : []; +} + +function relationObjectId(value: unknown): string | number | undefined { + if (value == null || typeof value != 'object' || Array.isArray(value)) { + return undefined; + } + const objectId = (value as Record).object_id; + return typeof objectId == 'string' || typeof objectId == 'number' ? objectId : undefined; +} + +function jsonEquals(left: unknown, right: unknown): boolean { + return JSON.stringify(left ?? null) == JSON.stringify(right ?? null); +} + +function maxBigint(...values: (bigint | null | undefined)[]): bigint { + return values.reduce((max, value) => (value != null && value > max ? value : max), 0n); +} + +function* chunked(items: T[], size: number): Generator { + for (let index = 0; index < items.length; index += size) { + yield items.slice(index, index + size); + } +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts new file mode 100644 index 000000000..91c463124 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts @@ -0,0 +1,247 @@ +import { MikroORM } from '@mikro-orm/core'; +import { DO_NOT_LOG } from '@powersync/lib-services-framework'; +import { GetIntanceOptions, storage } from '@powersync/service-core'; +import crypto from 'crypto'; +import * as uuid from 'uuid'; +import type { BucketData, BucketParameters, CurrentData, SyncRules } from '../entities/entities-index.js'; +import { MikroOrmPersistedReplicationStream } from './MikroOrmPersistedReplicationStream.js'; +import { MikroOrmCheckpointWatcher, MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; +import { MikroOrmSyncRulesStorage } from './MikroOrmSyncRulesStorage.js'; + +export interface MikroOrmBucketStorageFactoryOptions { + orm: MikroORM; + dialect: MikroOrmStorageDialect; + slotNamePrefix: string; +} + +export class MikroOrmBucketStorageFactory extends storage.BucketStorageFactory { + [DO_NOT_LOG] = true; + + readonly orm: MikroORM; + readonly dialect: MikroOrmStorageDialect; + readonly slotNamePrefix: string; + readonly checkpointWatcher: MikroOrmCheckpointWatcher; + + constructor(options: MikroOrmBucketStorageFactoryOptions) { + super(); + this.orm = options.orm; + this.dialect = options.dialect; + this.slotNamePrefix = options.slotNamePrefix; + this.checkpointWatcher = options.dialect.createCheckpointWatcher(); + } + + async [Symbol.asyncDispose](): Promise { + await this.orm.close(true); + } + + getInstance( + replicationStream: storage.PersistedReplicationStream, + options?: GetIntanceOptions + ): storage.SyncRulesBucketStorage { + const syncRuleStorage = new MikroOrmSyncRulesStorage({ + factory: this, + orm: this.orm, + dialect: this.dialect, + replicationStream + }); + + if (!options?.skipLifecycleHooks) { + this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage)); + } + + syncRuleStorage.registerListener({ + batchStarted: (batch) => { + batch.registerListener({ + replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) + }); + } + }); + + return syncRuleStorage; + } + + async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { + const storageVersion = + options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; + const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion]; + if (storageConfig == null) { + throw new Error(`Unsupported storage version ${storageVersion}`); + } + + const em = this.orm.em.fork(); + let row: SyncRules; + await em.transactional(async (transactionalEntityManager) => { + const processingRows = await transactionalEntityManager.find(this.dialect.syncRulesEntity, { + state: storage.SyncRuleState.PROCESSING + }); + for (const processingRow of processingRows) { + transactionalEntityManager.assign(processingRow, { state: storage.SyncRuleState.STOP }); + } + + const syncRules = transactionalEntityManager.create(this.dialect.syncRulesEntity, { + state: storage.SyncRuleState.PROCESSING, + snapshotDone: false, + snapshotLsn: null, + lastCheckpoint: null, + lastCheckpointLsn: null, + noCheckpointBefore: null, + slotName: this.generateReplicationStreamName(), + lastCheckpointTs: null, + lastKeepaliveTs: null, + lastFatalError: null, + lastFatalErrorTs: null, + keepaliveOp: null, + storageVersion, + content: options.config.yaml, + syncPlan: options.config.plan + }); + + transactionalEntityManager.persist(syncRules); + await transactionalEntityManager.flush(); + row = syncRules; + }); + + return new MikroOrmPersistedReplicationStream(this.orm, this.dialect, row!); + } + + async restartReplication(replicationStreamId: number): Promise { + const active = await this.getActiveSyncConfig(); + const deploying = await this.getDeployingSyncConfig(); + const stream = + deploying?.replicationStream.replicationStreamId == replicationStreamId + ? deploying + : active?.replicationStream.replicationStreamId == replicationStreamId + ? active + : null; + + if (stream == null) { + return; + } + + await this.updateSyncRules(stream.content.asUpdateOptions()); + } + + async getActiveSyncConfig(): Promise { + return this.getSyncConfigForStates([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED]); + } + + async getDeployingSyncConfig(): Promise { + return this.getSyncConfigForStates([storage.SyncRuleState.PROCESSING]); + } + + async getReplicatingReplicationStreams(): Promise { + const rows = await this.findSyncRulesRows([storage.SyncRuleState.PROCESSING]); + return rows.map((row) => new MikroOrmPersistedReplicationStream(this.orm, this.dialect, row)); + } + + async getStoppedReplicationStreams(): Promise { + const rows = await this.findSyncRulesRows([storage.SyncRuleState.STOP]); + return rows.map((row) => new MikroOrmPersistedReplicationStream(this.orm, this.dialect, row)); + } + + async getStorageMetrics(): Promise { + const em = this.orm.em.fork(); + const [operations, parameters, currentData] = await Promise.all([ + em.find(this.dialect.bucketDataEntity, {}), + em.find(this.dialect.bucketParametersEntity, {}), + em.find(this.dialect.currentDataEntity, {}) + ]); + + return { + operations_size_bytes: operations.reduce((total, row) => total + estimateBucketDataSize(row), 0), + parameters_size_bytes: parameters.reduce((total, row) => total + estimateBucketParametersSize(row), 0), + replication_size_bytes: currentData.reduce((total, row) => total + estimateCurrentDataSize(row), 0) + }; + } + + async getPowerSyncInstanceId(): Promise { + const em = this.orm.em.fork(); + const instanceEntity = this.dialect.instanceEntity; + const [existingRow] = await em.find(instanceEntity, {}, { limit: 1 }); + let row = existingRow ?? null; + if (row == null) { + row = em.create(instanceEntity, { id: uuid.v4() }); + await em.persist(row).flush(); + } + return row.id; + } + + async getSystemIdentifier(): Promise { + return { + id: `${this.dialect.type}:${await this.getPowerSyncInstanceId()}`, + type: this.dialect.type + }; + } + + private async getSyncConfigForStates(states: storage.SyncRuleState[]): Promise { + const [row] = await this.findSyncRulesRows(states); + if (row == null) { + return null; + } + + const replicationStream = new MikroOrmPersistedReplicationStream(this.orm, this.dialect, row); + const storageInstance = this.getInstance(replicationStream, { skipLifecycleHooks: true }); + return { + content: replicationStream.syncConfigContent[0], + replicationStream, + storage: storageInstance + }; + } + + private async findSyncRulesRows(states: storage.SyncRuleState[]): Promise { + const em = this.orm.em.fork(); + return await em.find(this.dialect.syncRulesEntity, { state: { $in: states } }, { orderBy: { id: 'DESC' } }); + } + + private generateReplicationStreamName(): string { + return `${this.slotNamePrefix}${Date.now()}_${crypto.randomBytes(2).toString('hex')}`; + } +} + +function estimateString(value: string | null | undefined): number { + return value == null ? 0 : value.length; +} + +function estimateBuffer(value: Buffer | Uint8Array | null | undefined): number { + return value == null ? 0 : value.byteLength; +} + +function estimateJson(value: unknown): number { + return typeof value == 'string' ? value.length : JSON.stringify(value ?? null).length; +} + +function estimateBucketDataSize(row: BucketData): number { + return ( + 80 + + estimateString(row.id) + + estimateString(row.bucketName) + + estimateString(row.op) + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateString(row.tableName) + + estimateString(row.rowId) + + estimateString(row.data) + ); +} + +function estimateBucketParametersSize(row: BucketParameters): number { + return ( + 80 + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateBuffer(row.lookup) + + estimateJson(row.bucketParameters) + ); +} + +function estimateCurrentDataSize(row: CurrentData): number { + return ( + 80 + + estimateString(row.id) + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateJson(row.buckets) + + estimateJson(row.lookups) + + estimateBuffer(row.data) + ); +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmCompactor.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmCompactor.ts new file mode 100644 index 000000000..3939ea973 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmCompactor.ts @@ -0,0 +1,498 @@ +import type { MikroORM } from '@mikro-orm/core'; +import { Logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import * as uuid from 'uuid'; +import type { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +interface CurrentBucketState { + bucket: string; + seen: Map; + trackingSize: number; + lastNotPut: bigint | null; + opsSincePut: number; +} + +interface ParameterCompactionRow { + id: bigint; + sourceTable: string; + sourceKey: Buffer; + lookup: Buffer; + bucketParameters: unknown; +} + +interface RawParameterCompactionRow { + id: bigint | number | string; + source_table?: string; + sourceTable?: string; + source_key?: Buffer | Uint8Array; + sourceKey?: Buffer | Uint8Array; + lookup: Buffer | Uint8Array; + bucket_parameters?: unknown; + bucketParameters?: unknown; +} + +interface RawBucketDataRow { + id: string; + bucket_name: string; + op_id: bigint | number | string; + op: string; + source_table: string | null; + source_key: Buffer | Uint8Array | null; + table_name: string | null; + row_id: string | null; + checksum: bigint | number | string; + target_op: bigint | number | string | null; +} + +interface CompactionBucketDataRow { + id: string; + bucketName: string; + opId: bigint; + op: string; + sourceTable: string | null; + sourceKey: Buffer | null; + tableName: string | null; + rowId: string | null; + checksum: bigint; + targetOp: bigint | null; +} + +export interface MikroOrmCompactOptions extends storage.CompactOptions { + logger: Logger; +} + +const BIGINT_MAX = 9223372036854775807n; +const DEFAULT_CLEAR_BATCH_LIMIT = 5000; +const DEFAULT_MOVE_BATCH_LIMIT = 2000; +const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; +const DEFAULT_MEMORY_LIMIT_MB = 64; +const PARAMETER_DELETE_BATCH_LIMIT = 1000; +const PARAMETER_SCAN_BATCH_LIMIT = 10_000; + +export class MikroOrmCompactor { + private readonly idLimitBytes: number; + private readonly moveBatchLimit: number; + private readonly moveBatchQueryLimit: number; + private readonly clearBatchLimit: number; + private readonly maxOpId: bigint; + private readonly buckets: string[] | undefined; + private readonly logger: Logger; + + private pendingMoves: { id: string; targetOp: bigint }[] = []; + + constructor( + private readonly orm: MikroORM, + private readonly dialect: MikroOrmStorageDialect, + private readonly groupId: number, + options: MikroOrmCompactOptions + ) { + this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; + this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; + this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; + this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; + this.maxOpId = options.maxOpId ?? 0n; + this.buckets = options.compactBuckets; + this.logger = options.logger; + } + + async compact(): Promise { + if (this.maxOpId <= 0n) { + return; + } + + if (this.buckets != null) { + for (const bucket of this.buckets) { + await this.compactSingleBucket(bucket); + } + } else { + await this.compactAllBuckets(); + } + } + + async compactParameterData(options: storage.CompactOptions): Promise { + if (this.maxOpId <= 0n) { + return; + } + + const lastByKey = new Map(); + const removeIds = new Set(); + const keysWithDuplicateValues = new Set(); + const maxCacheSize = options.compactParameterCacheLimit ?? 10_000; + const flushDeletes = async (force: boolean) => { + if (removeIds.size < PARAMETER_DELETE_BATCH_LIMIT && !(force && removeIds.size > 0)) { + return; + } + + const removedCount = removeIds.size; + for (const id of removeIds) { + await this.executeRun( + ` + DELETE FROM bucket_parameters + WHERE group_id = ? AND id = ? + `, + [this.groupId, id.toString()] + ); + } + removeIds.clear(); + this.logger.info(`Removed ${removedCount} compacted parameter entries`); + }; + + for await (const row of this.streamParameterRows()) { + const key = parameterKey(row); + const previous = lastByKey.get(key); + if (previous != null && sameBucketParameters(previous.bucketParameters, row.bucketParameters)) { + removeIds.add(row.id); + keysWithDuplicateValues.add(key); + } + + if (isEmptyBucketParameters(row.bucketParameters) && row.id < this.maxOpId && !keysWithDuplicateValues.has(key)) { + await flushDeletes(true); + const candidate = lastByKey.get(key); + await this.executeRun( + ` + DELETE FROM bucket_parameters + WHERE group_id = ? + AND lookup = ? + AND source_table = ? + AND source_key = ? + AND id <= ? + `, + [this.groupId, row.lookup, row.sourceTable, row.sourceKey, row.id.toString()] + ); + if (candidate != null && candidate.id <= row.id) { + removeIds.add(candidate.id); + } + removeIds.add(row.id); + lastByKey.delete(key); + } else { + lastByKey.set(key, row); + } + + if (lastByKey.size > maxCacheSize) { + const oldest = lastByKey.keys().next().value; + if (oldest != null) { + lastByKey.delete(oldest); + } + } + + await flushDeletes(false); + } + await flushDeletes(true); + lastByKey.clear(); + } + + private async *streamParameterRows(): AsyncIterable { + let lastRow: ParameterCompactionRow | null = null; + + while (true) { + const params: unknown[] = [this.groupId, this.maxOpId]; + let cursorFilter = ''; + if (lastRow != null) { + cursorFilter = ` + AND ( + lookup > ? + OR (lookup = ? AND source_table > ?) + OR (lookup = ? AND source_table = ? AND source_key > ?) + OR (lookup = ? AND source_table = ? AND source_key = ? AND id > ?) + ) + `; + params.push( + lastRow.lookup, + lastRow.lookup, + lastRow.sourceTable, + lastRow.lookup, + lastRow.sourceTable, + lastRow.sourceKey, + lastRow.lookup, + lastRow.sourceTable, + lastRow.sourceKey, + lastRow.id + ); + } + params.push(PARAMETER_SCAN_BATCH_LIMIT); + + const rows = await this.executeAll( + ` + SELECT id, source_table, source_key, lookup, bucket_parameters + FROM bucket_parameters + WHERE group_id = ? + AND id <= ? + ${cursorFilter} + ORDER BY lookup ASC, source_table ASC, source_key ASC, id ASC + LIMIT ? + `, + params + ); + + if (rows.length == 0) { + return; + } + + for (const rawRow of rows) { + lastRow = rawParameterRow(rawRow); + yield lastRow; + } + } + } + + private async compactAllBuckets(): Promise { + const discoveryBatchSize = 200; + let lastBucket = ''; + + while (true) { + const rows = await this.executeAll<{ bucket_name: string }>( + ` + SELECT DISTINCT bucket_name + FROM bucket_data + WHERE group_id = ? AND bucket_name > ? + ORDER BY bucket_name ASC + LIMIT ? + `, + [this.groupId, lastBucket, discoveryBatchSize] + ); + if (rows.length == 0) { + break; + } + + for (const row of rows) { + await this.compactSingleBucket(row.bucket_name); + } + lastBucket = rows[rows.length - 1].bucket_name; + } + } + + private async compactSingleBucket(bucket: string): Promise { + const currentState: CurrentBucketState = { + bucket, + seen: new Map(), + trackingSize: 0, + lastNotPut: null, + opsSincePut: 0 + }; + let upperOpIdLimit = BIGINT_MAX; + + while (true) { + const batch = ( + await this.executeAll( + ` + SELECT id, bucket_name, op_id, op, source_table, source_key, table_name, row_id, checksum, target_op + FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id < ? + AND op_id <= ? + ORDER BY op_id DESC + LIMIT ? + `, + [this.groupId, bucket, upperOpIdLimit, this.maxOpId, this.moveBatchQueryLimit] + ) + ).map(rawBucketDataRow); + + if (batch.length == 0) { + break; + } + + upperOpIdLimit = batch[batch.length - 1].opId; + + for (const row of batch) { + let isPersistentPut = row.op == 'PUT'; + + if (row.op == 'REMOVE' || row.op == 'PUT') { + const key = compactBucketRowKey(row); + const targetOp = currentState.seen.get(utils.flatstr(key)); + if (targetOp != null) { + isPersistentPut = false; + this.pendingMoves.push({ id: row.id, targetOp }); + } else if (currentState.trackingSize < this.idLimitBytes) { + currentState.seen.set(utils.flatstr(key), row.opId); + currentState.trackingSize += key.length + 140; + } + } + + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (row.op != 'CLEAR') { + currentState.lastNotPut ??= row.opId; + currentState.opsSincePut += 1; + } + + if (this.pendingMoves.length >= this.moveBatchLimit) { + await this.flushMoves(); + } + } + } + + await this.flushMoves(); + currentState.seen.clear(); + if (currentState.lastNotPut != null && currentState.opsSincePut > 1) { + this.logger.info( + `Inserting CLEAR at ${this.groupId}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + await this.clearBucket(currentState.bucket, currentState.lastNotPut); + } + } + + private async flushMoves(): Promise { + if (this.pendingMoves.length == 0) { + return; + } + + const batch = this.pendingMoves.splice(0); + this.logger.info(`Compacting ${batch.length} ops`); + for (const { id, targetOp } of batch) { + await this.executeRun( + ` + UPDATE bucket_data + SET op = 'MOVE', + target_op = ?, + table_name = NULL, + row_id = NULL, + data = NULL, + source_table = NULL, + source_key = NULL + WHERE id = ? + `, + [targetOp, id] + ); + } + } + + private async clearBucket(bucket: string, op: bigint): Promise { + let done = false; + while (!done) { + const operationRows = ( + await this.executeAll( + ` + SELECT id, bucket_name, op_id, op, source_table, source_key, table_name, row_id, checksum, target_op + FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id <= ? + ORDER BY op_id ASC + LIMIT ? + `, + [this.groupId, bucket, op, this.clearBatchLimit] + ) + ).map(rawBucketDataRow); + + let checksum = 0; + let lastOpId: bigint | null = null; + let targetOp: bigint | null = null; + let gotAnOp = false; + + for (const operation of operationRows) { + if (operation.op != 'MOVE' && operation.op != 'REMOVE' && operation.op != 'CLEAR') { + throw new Error(`Unexpected ${operation.op} operation at ${this.groupId}:${bucket}:${operation.opId}`); + } + + checksum = utils.addChecksums(checksum, Number(operation.checksum)); + lastOpId = operation.opId; + if (operation.op != 'CLEAR') { + gotAnOp = true; + } + if (operation.targetOp != null && (targetOp == null || operation.targetOp > targetOp)) { + targetOp = operation.targetOp; + } + } + + if (!gotAnOp || lastOpId == null) { + done = true; + return; + } + + this.logger.info(`Flushing CLEAR at ${lastOpId}`); + await this.executeRun( + ` + DELETE FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id <= ? + `, + [this.groupId, bucket, lastOpId] + ); + await this.executeRun( + ` + INSERT INTO bucket_data ( + id, + group_id, + bucket_name, + op_id, + op, + checksum, + target_op + ) VALUES (?, ?, ?, ?, 'CLEAR', ?, ?) + `, + [uuid.v4(), this.groupId, bucket, lastOpId, BigInt(checksum), targetOp] + ); + } + } + + private async executeAll(sql: string, params: unknown[]): Promise { + return this.orm.em.getConnection().execute(sql, params, 'all'); + } + + private async executeRun(sql: string, params: unknown[]): Promise { + await this.orm.em.getConnection().execute(sql, params, 'run'); + } +} + +function compactBucketRowKey(row: CompactionBucketDataRow): string { + return `${row.tableName}/${row.rowId}/${row.sourceTable}.${row.sourceKey == null ? '' : row.sourceKey.toString('base64')}`; +} + +function rawBucketDataRow(row: RawBucketDataRow): CompactionBucketDataRow { + return { + id: row.id, + bucketName: row.bucket_name, + opId: BigInt(row.op_id), + op: row.op, + sourceTable: row.source_table, + sourceKey: row.source_key == null ? null : Buffer.from(row.source_key), + tableName: row.table_name, + rowId: row.row_id, + checksum: BigInt(row.checksum), + targetOp: row.target_op == null ? null : BigInt(row.target_op) + }; +} + +function rawParameterRow(row: RawParameterCompactionRow): ParameterCompactionRow { + const sourceTable = row.source_table ?? row.sourceTable; + const sourceKey = row.source_key ?? row.sourceKey; + if (sourceTable == null || sourceKey == null) { + throw new Error('Expected parameter compaction row source columns'); + } + + return { + id: BigInt(row.id), + sourceTable, + sourceKey: Buffer.from(sourceKey), + lookup: Buffer.from(row.lookup), + bucketParameters: row.bucket_parameters ?? row.bucketParameters + }; +} + +function parameterKey(row: ParameterCompactionRow): string { + return `${row.lookup.toString('base64')}/${row.sourceTable}/${row.sourceKey.toString('base64')}`; +} + +function normalizedBucketParameters(value: unknown): string { + if (typeof value != 'string') { + return JSON.stringify(value ?? null); + } + + try { + const parsed = JSON.parse(value); + return typeof parsed == 'string' ? parsed : JSON.stringify(parsed); + } catch { + return value; + } +} + +function sameBucketParameters(left: unknown, right: unknown): boolean { + return normalizedBucketParameters(left) == normalizedBucketParameters(right); +} + +function isEmptyBucketParameters(value: unknown): boolean { + return normalizedBucketParameters(value) == '[]'; +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts new file mode 100644 index 000000000..2f3865ad2 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts @@ -0,0 +1,375 @@ +import type { EntityManager } from '@mikro-orm/core'; +import type { Logger } from '@powersync/lib-services-framework'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as uuid from 'uuid'; +import type { CurrentData } from '../entities/entities-index.js'; +import type { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +export interface MikroOrmPersistedBatchOptions { + transactionalEntityManager: EntityManager; + dialect: MikroOrmStorageDialect; + logger: Logger; + syncRules: sync_rules.HydratedSyncConfig; + replicationStreamId: number; + storeCurrentData: boolean; + skipExistingRows: boolean; + markRecordUnavailable: storage.BucketStorageMarkRecordUnavailable | undefined; +} + +export type CurrentBucket = { + bucket: string; + table: string; + id: string; +}; + +/** + * Handles the writes for a single persisted operation chunk. + * + * `MikroOrmBucketBatch` owns the public batch lifecycle, checkpointing, and + * listener hooks. This class owns the transactional write state for bucket + * data, parameter rows, and current data so large flushes can be split into + * smaller persisted chunks without keeping every tracked entity in one unit of + * work. + */ +export class MikroOrmPersistedBatch { + private readonly currentDataById = new Map(); + + constructor(private readonly options: MikroOrmPersistedBatchOptions) {} + + async persistOperations(operations: storage.SaveOptions[], nextOpId: bigint): Promise { + await this.loadCurrentData(operations); + for (const operation of operations) { + nextOpId = await this.persistOperation(operation, nextOpId); + } + return nextOpId; + } + + persistBucketData(options: { + table: storage.SourceTable; + sourceKey: storage.ReplicaId; + existingBuckets: CurrentBucket[]; + evaluated: sync_rules.EvaluatedRow[]; + nextOpId: bigint; + }): bigint { + const remainingBuckets = new Map(options.existingBuckets.map((bucket) => [currentBucketKey(bucket), bucket])); + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + const deleteChecksum = utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)); + let nextOpId = options.nextOpId; + + for (const row of options.evaluated) { + remainingBuckets.delete(currentBucketKey(row)); + const data = JSONBig.stringify(row.data); + const checksum = utils.hashData(row.table, row.id, data); + this.options.transactionalEntityManager.persist( + this.options.transactionalEntityManager.create(this.options.dialect.bucketDataEntity, { + id: uuid.v4(), + groupId: this.options.replicationStreamId, + bucketName: row.bucket, + opId: nextOpId++, + op: 'PUT', + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + tableName: row.table, + rowId: row.id, + checksum: BigInt(checksum), + data, + targetOp: null + }) + ); + } + + for (const bucket of remainingBuckets.values()) { + this.options.transactionalEntityManager.persist( + this.options.transactionalEntityManager.create(this.options.dialect.bucketDataEntity, { + id: uuid.v4(), + groupId: this.options.replicationStreamId, + bucketName: bucket.bucket, + opId: nextOpId++, + op: 'REMOVE', + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + tableName: bucket.table, + rowId: bucket.id, + checksum: BigInt(deleteChecksum), + data: null, + targetOp: null + }) + ); + } + + return nextOpId; + } + + persistParameterData(options: { + table: storage.SourceTable; + sourceKey: storage.ReplicaId; + existingLookups: Buffer[]; + evaluated: sync_rules.EvaluatedParameters[]; + nextOpId: bigint; + }): bigint { + const remainingLookups = new Map(options.existingLookups.map((lookup) => [lookup.toString('base64'), lookup])); + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + let nextOpId = options.nextOpId; + + for (const row of options.evaluated) { + const lookup = storage.serializeLookupBuffer(row.lookup); + remainingLookups.delete(lookup.toString('base64')); + this.options.transactionalEntityManager.persist( + this.options.transactionalEntityManager.create(this.options.dialect.bucketParametersEntity, { + id: nextOpId++, + groupId: this.options.replicationStreamId, + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + lookup, + bucketParameters: JSONBig.stringify(row.bucketParameters) + }) + ); + } + + for (const lookup of remainingLookups.values()) { + this.options.transactionalEntityManager.persist( + this.options.transactionalEntityManager.create(this.options.dialect.bucketParametersEntity, { + id: nextOpId++, + groupId: this.options.replicationStreamId, + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + lookup, + bucketParameters: '[]' + }) + ); + } + + return nextOpId; + } + + private async persistOperation(record: storage.SaveOptions, nextOpId: bigint): Promise { + const sourceTable = record.sourceTable; + const tableId = String(sourceTable.id); + const afterId = record.afterReplicaId ?? null; + const beforeId = record.beforeReplicaId ?? record.afterReplicaId; + const serializedBeforeId = storage.serializeReplicaId(beforeId); + const existingCurrentDataId = currentDataId(this.options.replicationStreamId, tableId, serializedBeforeId); + const existingCurrentData = this.currentDataById.get(existingCurrentDataId) ?? null; + + const storeCurrentData = this.options.storeCurrentData && sourceTable.storeCurrentData; + let existingBuckets = currentBuckets(existingCurrentData); + let existingLookups = currentLookups(existingCurrentData); + let after = record.after; + + if (this.options.skipExistingRows) { + if (record.tag == storage.SaveOperationTag.INSERT) { + if (existingCurrentData != null) { + return nextOpId; + } + } else { + throw new ReplicationAssertionError(`${record.tag} not supported with skipExistingRows: true`); + } + } + + if (record.tag == storage.SaveOperationTag.UPDATE) { + if (existingCurrentData != null && storeCurrentData) { + after = storage.mergeToast(record.after, storage.deserializeBson(Buffer.from(existingCurrentData.data))); + } else if (existingCurrentData == null && storeCurrentData) { + this.options.markRecordUnavailable?.(record); + } + } + + if (beforeId != null && (afterId == null || !storage.replicaIdEquals(beforeId, afterId))) { + if (sourceTable.syncData) { + nextOpId = this.persistBucketData({ + table: sourceTable, + sourceKey: beforeId, + existingBuckets, + evaluated: [], + nextOpId + }); + existingBuckets = []; + } + + if (sourceTable.syncParameters) { + nextOpId = this.persistParameterData({ + table: sourceTable, + sourceKey: beforeId, + existingLookups, + evaluated: [], + nextOpId + }); + existingLookups = []; + } + } + + let newBuckets: CurrentBucket[] = []; + let newLookups: Buffer[] = []; + let afterData: Buffer | undefined; + if (afterId != null && after != null && utils.isCompleteRow(storeCurrentData, after)) { + afterData = storeCurrentData ? storage.serializeBson(after) : storage.serializeBson({}); + + if (sourceTable.syncData) { + const { results, errors } = this.options.syncRules.evaluateRowWithErrors({ + record: after, + sourceTable: sourceTable.ref, + bucketDataSources: sourceTable.bucketDataSources + }); + for (const error of errors) { + this.options.logger.error( + `Failed to evaluate data query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + nextOpId = this.persistBucketData({ + table: sourceTable, + sourceKey: afterId, + existingBuckets, + evaluated: results, + nextOpId + }); + newBuckets = results.map((row) => ({ + bucket: row.bucket, + table: row.table, + id: row.id + })); + } + + if (sourceTable.syncParameters) { + const { results, errors } = this.options.syncRules.evaluateParameterRowWithErrors(sourceTable.ref, after, { + parameterLookupSources: sourceTable.parameterLookupSources + }); + for (const error of errors) { + this.options.logger.error( + `Failed to evaluate parameter query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + nextOpId = this.persistParameterData({ + table: sourceTable, + sourceKey: afterId, + existingLookups, + evaluated: results, + nextOpId + }); + newLookups = results.map((row) => storage.serializeLookupBuffer(row.lookup)); + } + } + + if (afterId != null && afterData != null) { + await this.upsertCurrentData({ + tableId, + sourceKey: afterId, + buckets: newBuckets, + lookups: newLookups, + data: afterData, + pendingDelete: null + }); + } + + if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { + nextOpId = await this.deleteCurrentData(tableId, beforeId, nextOpId); + } + + return nextOpId; + } + + private async loadCurrentData(operations: storage.SaveOptions[]): Promise { + const ids = new Set(); + for (const operation of operations) { + const tableId = String(operation.sourceTable.id); + const beforeId = operation.beforeReplicaId ?? operation.afterReplicaId; + ids.add(currentDataId(this.options.replicationStreamId, tableId, storage.serializeReplicaId(beforeId))); + + const afterId = operation.afterReplicaId ?? null; + if (afterId != null) { + ids.add(currentDataId(this.options.replicationStreamId, tableId, storage.serializeReplicaId(afterId))); + } + } + + const rows = + ids.size == 0 + ? [] + : await this.options.transactionalEntityManager.find(this.options.dialect.currentDataEntity, { + id: { $in: [...ids] } + }); + for (const row of rows) { + this.currentDataById.set(row.id, row); + } + } + + private async upsertCurrentData(options: { + tableId: string; + sourceKey: storage.ReplicaId; + buckets: CurrentBucket[]; + lookups: Buffer[]; + data: Buffer; + pendingDelete: bigint | null; + }): Promise { + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + const id = currentDataId(this.options.replicationStreamId, options.tableId, serializedSourceKey); + const payload = { + id, + groupId: this.options.replicationStreamId, + sourceTable: options.tableId, + sourceKey: serializedSourceKey, + buckets: options.buckets, + lookups: options.lookups.map((lookup) => lookup.toString('hex')), + data: options.data, + pendingDelete: options.pendingDelete + }; + const existing = this.currentDataById.get(id) ?? null; + if (existing == null) { + const row = this.options.transactionalEntityManager.create(this.options.dialect.currentDataEntity, payload); + this.options.transactionalEntityManager.persist(row); + this.currentDataById.set(id, row); + } else { + this.options.transactionalEntityManager.assign(existing, payload); + this.currentDataById.set(id, existing); + } + } + + private async deleteCurrentData(tableId: string, sourceKey: storage.ReplicaId, nextOpId: bigint): Promise { + const serializedSourceKey = storage.serializeReplicaId(sourceKey); + const id = currentDataId(this.options.replicationStreamId, tableId, serializedSourceKey); + const payload = { + id, + groupId: this.options.replicationStreamId, + sourceTable: tableId, + sourceKey: serializedSourceKey, + buckets: [], + lookups: [], + data: storage.serializeBson({}), + pendingDelete: nextOpId + }; + const existing = this.currentDataById.get(id) ?? null; + if (existing == null) { + const row = this.options.transactionalEntityManager.create(this.options.dialect.currentDataEntity, payload); + this.options.transactionalEntityManager.persist(row); + this.currentDataById.set(id, row); + } else { + this.options.transactionalEntityManager.assign(existing, payload); + this.currentDataById.set(id, existing); + } + return nextOpId + 1n; + } +} + +export function currentBuckets(row: CurrentData | null): CurrentBucket[] { + return Array.isArray(row?.buckets) ? (row.buckets as CurrentBucket[]) : []; +} + +export function currentLookups(row: CurrentData | null): Buffer[] { + return Array.isArray(row?.lookups) ? (row.lookups as string[]).map((lookup) => Buffer.from(lookup, 'hex')) : []; +} + +function currentDataId(groupId: number, sourceTable: string, sourceKey: Buffer): string { + return `${groupId}:${sourceTable}:${sourceKey.toString('hex')}`; +} + +function currentBucketKey(bucket: CurrentBucket | sync_rules.EvaluatedRow): string { + return `${bucket.bucket}/${bucket.table}/${bucket.id}`; +} + +function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string { + if (storage.isUUID(id)) { + return `${tableId}/${id.toHexString()}`; + } + return uuid.v5(storage.serializeBson({ table: tableId, id }), utils.ID_NAMESPACE); +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts new file mode 100644 index 000000000..a0ecc6eb1 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts @@ -0,0 +1,81 @@ +import { MikroORM } from '@mikro-orm/core'; +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import type { SyncRules } from '../entities/entities-index.js'; +import { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +export class MikroOrmPersistedSyncConfigContent extends storage.PersistedSyncConfigContent { + constructor( + private readonly orm: MikroORM, + private readonly dialect: MikroOrmStorageDialect, + row: SyncRules + ) { + super({ + replicationStreamId: Number(row.id), + sync_rules_content: row.content, + compiled_plan: row.syncPlan, + replicationStreamName: row.slotName, + storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION, + syncConfigId: String(row.id) + }); + } + + async getSyncConfigStatus(): Promise { + const em = this.orm.em.fork(); + const row = await em.findOne(this.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + + return row == null ? null : syncConfigStatusFromRow(row); + } +} + +export class MikroOrmPersistedReplicationStream extends storage.PersistedReplicationStream { + current_lock: storage.ReplicationLock | null = null; + readonly syncConfigContent: readonly MikroOrmPersistedSyncConfigContent[]; + + constructor( + private readonly orm: MikroORM, + private readonly dialect: MikroOrmStorageDialect, + private readonly row: SyncRules + ) { + super({ + replicationStreamId: Number(row.id), + replicationStreamName: row.slotName, + state: row.state, + storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION + }); + this.syncConfigContent = [new MikroOrmPersistedSyncConfigContent(this.orm, this.dialect, this.row)]; + } + + parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet { + return this.syncConfigContent[0].parsed(options); + } + + async lock(): Promise { + if (this.current_lock != null) { + throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by this process.`); + } + + return (this.current_lock = { + sync_rules_id: this.replicationStreamId, + release: async () => { + this.current_lock = null; + } + }); + } +} + +export function syncConfigStatusFromRow(row: SyncRules): storage.PersistedSyncConfigStatus { + return { + id: String(row.id), + replicationStreamId: Number(row.id), + state: row.state, + snapshot_done: row.snapshotDone, + last_checkpoint_lsn: row.lastCheckpointLsn, + last_fatal_error: row.lastFatalError, + last_fatal_error_ts: row.lastFatalErrorTs, + last_keepalive_ts: row.lastKeepaliveTs, + last_checkpoint_ts: row.lastCheckpointTs + }; +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmReportStorage.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmReportStorage.ts new file mode 100644 index 000000000..ccd623e8a --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmReportStorage.ts @@ -0,0 +1,34 @@ +import { storage } from '@powersync/service-core'; +import { event_types } from '@powersync/service-types'; + +export class MikroOrmReportStorage implements storage.ReportStorage { + async [Symbol.asyncDispose](): Promise { + // Report storage is intentionally a no-op in the initial MikroORM storage slice. + } + + async reportClientConnection(_data: event_types.ClientConnectionBucketData): Promise {} + + async reportClientDisconnection(_data: event_types.ClientDisconnectionEventData): Promise {} + + async getConnectedClients(): Promise { + return { users: [], sdks: [] } as unknown as event_types.ClientConnectionReportResponse; + } + + async getClientConnectionReports( + _data: event_types.ClientConnectionReportRequest + ): Promise { + return { users: [], sdks: [] } as unknown as event_types.ClientConnectionReportResponse; + } + + async getGeneralClientConnectionAnalytics( + _data: event_types.ClientConnectionAnalyticsRequest + ): Promise> { + return { + items: [], + count: 0, + more: false + } as unknown as event_types.PaginatedResponse; + } + + async deleteOldConnectionData(_data: event_types.DeleteOldConnectionData): Promise {} +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts new file mode 100644 index 000000000..0678cce03 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts @@ -0,0 +1,65 @@ +import type { EntityClass, EntityManager } from '@mikro-orm/core'; +import type { BucketDataRequest } from '@powersync/service-core'; +import type { + BucketData, + BucketParameters, + CurrentData, + Instance, + SourceTable, + SyncRules, + WriteCheckpoint +} from '../entities/entities-index.js'; + +/** + * Complete database-specific adapter consumed by the common MikroORM storage classes. + */ +export interface MikroOrmStorageDialect { + /** Public storage type identifier, e.g. `mikroorm:sqlite`. */ + readonly type: string; + /** Flat entity class list passed to MikroORM initialization and migration tooling. */ + readonly entityClasses: EntityClass[]; + /** Stored bucket operations, typically read by bucket and operation id. */ + readonly bucketDataEntity: EntityClass; + /** Materialized parameter lookups for bucket parameter queries. */ + readonly bucketParametersEntity: EntityClass; + /** Latest known source row state used by replication batching and compaction paths. */ + readonly currentDataEntity: EntityClass; + /** Singleton service instance identity row. */ + readonly instanceEntity: EntityClass; + /** Source table replication metadata and snapshot progress. */ + readonly sourceTableEntity: EntityClass; + /** Sync rule versions and replication stream state. */ + readonly syncRulesEntity: EntityClass; + /** User write checkpoint rows. */ + readonly writeCheckpointEntity: EntityClass; + /** + * Stream bucket operation rows for sync reads. + * + * Hot bucket reads need to avoid buffering a whole result set before chunking. SQL drivers can implement this with + * MikroORM query builder streaming, while non-SQL drivers can provide their closest cursor equivalent. + */ + streamBucketDataRows(options: MikroOrmBucketDataStreamOptions): AsyncIterable; + /** Create a checkpoint watcher suitable for this database's notification capabilities. */ + createCheckpointWatcher(): MikroOrmCheckpointWatcher; +} + +export interface MikroOrmBucketDataStreamOptions { + readonly em: EntityManager; + readonly groupId: number; + readonly checkpoint: bigint; + readonly dataBuckets: BucketDataRequest[]; + readonly limit: number; +} + +/** + * Minimal checkpoint notification abstraction. + * + * SQLite can only notify listeners inside the current process, while future drivers can map this onto database-level + * notifications that work across service instances. + */ +export interface MikroOrmCheckpointWatcher { + /** Wake local watchers after this process writes a relevant checkpoint. */ + notify(): void; + /** Yield whenever a checkpoint change should be re-read. */ + watch(signal: AbortSignal): AsyncIterable; +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts new file mode 100644 index 000000000..4ccc63143 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts @@ -0,0 +1,70 @@ +import { logger } from '@powersync/lib-services-framework'; +import { storage, system } from '@powersync/service-core'; +import { createSqliteMikroOrm } from '../drivers/sqlite/sqlite-config.js'; +import { createSqliteMikroOrmStorageFactory } from '../drivers/sqlite/SqliteMikroOrmStorageFactory.js'; +import { + isMikroOrmSqliteStorageConfig, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + MikroOrmSqliteStorageConfig, + normalizeMikroOrmSqliteStorageConfig +} from '../types/types.js'; +import { MikroOrmReportStorage } from './MikroOrmReportStorage.js'; + +export class MikroOrmStorageProvider implements storage.StorageProvider { + get type() { + return MIKRO_ORM_SQLITE_STORAGE_TYPE; + } + + async getStorage(options: storage.GetStorageOptions): Promise { + const { storage: storageConfig } = options.resolvedConfig; + + if (!isMikroOrmSqliteStorageConfig(storageConfig)) { + throw new Error(`Cannot create MikroORM SQLite storage with provided config ${storageConfig.type}`); + } + assertSqliteServiceMode(options.serviceMode); + + const decodedConfig = MikroOrmSqliteStorageConfig.decode(storageConfig); + const normalizedConfig = normalizeMikroOrmSqliteStorageConfig(decodedConfig); + const orm = await createSqliteMikroOrm(normalizedConfig); + await orm.schema.update(); + + const storageFactory = await createSqliteMikroOrmStorageFactory({ + config: normalizedConfig, + slotNamePrefix: options.resolvedConfig.slot_name_prefix, + orm + }); + const reportStorage = new MikroOrmReportStorage(); + + return { + reportStorage, + storage: storageFactory, + shutDown: async () => { + await storageFactory[Symbol.asyncDispose](); + }, + tearDown: async () => { + logger.info(`Tearing down MikroORM SQLite storage: ${normalizedConfig.filename}...`); + await orm.schema.drop(); + await storageFactory[Symbol.asyncDispose](); + return true; + } + }; + } +} + +const SQLITE_ALLOWED_COMMAND_MODES = new Set([ + system.ServiceContextMode.COMPACT, + system.ServiceContextMode.TEARDOWN, + system.ServiceContextMode.TEST_CONNECTION +]); + +function assertSqliteServiceMode(serviceMode: string): void { + if (serviceMode == system.ServiceContextMode.UNIFIED || SQLITE_ALLOWED_COMMAND_MODES.has(serviceMode)) { + return; + } + + throw new Error( + `MikroORM SQLite storage only supports the unified service runner. ` + + `SQLite checkpoint notifications are process-local, so split "${system.ServiceContextMode.API}" and "${system.ServiceContextMode.SYNC}" runners cannot safely share this storage. ` + + `Start the service with runner type "${system.ServiceContextMode.UNIFIED}" instead of "${serviceMode}".` + ); +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts new file mode 100644 index 000000000..8aa915930 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts @@ -0,0 +1,594 @@ +import { MikroORM } from '@mikro-orm/core'; +import { BaseObserver, DO_NOT_LOG, errors, Logger } from '@powersync/lib-services-framework'; +import { + BucketChecksumRequest, + BucketDataBatchOptions, + BucketDataRequest, + CHECKPOINT_INVALIDATE_ALL, + CheckpointChanges, + GetCheckpointChangesOptions, + PopulateChecksumCacheOptions, + PopulateChecksumCacheResults, + ReplicationCheckpoint, + storage, + StorageCheckpointUpdate, + SyncBucketDataChunk, + utils, + WatchWriteCheckpointOptions +} from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as uuid from 'uuid'; +import type { BucketData } from '../entities/entities-index.js'; +import { MikroOrmBucketBatch } from './MikroOrmBucketBatch.js'; +import { MikroOrmBucketStorageFactory } from './MikroOrmBucketStorageFactory.js'; +import { MikroOrmCompactor } from './MikroOrmCompactor.js'; +import { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +export interface MikroOrmSyncRulesStorageOptions { + factory: MikroOrmBucketStorageFactory; + orm: MikroORM; + dialect: MikroOrmStorageDialect; + replicationStream: storage.PersistedReplicationStream; +} + +export class MikroOrmSyncRulesStorage + extends BaseObserver + implements storage.SyncRulesBucketStorage +{ + [DO_NOT_LOG] = true; + + readonly replicationStreamId: number; + readonly replicationStreamName: string; + readonly storageConfig: storage.StorageVersionConfig; + readonly factory: MikroOrmBucketStorageFactory; + readonly logger: Logger; + + private parsedSyncRulesCache: + | { parsed: sync_rules.HydratedSyncConfig; options: storage.ParseSyncConfigOptions } + | undefined; + + private writeCheckpointModeValue = storage.WriteCheckpointMode.MANAGED; + + constructor(private readonly options: MikroOrmSyncRulesStorageOptions) { + super(); + this.replicationStreamId = options.replicationStream.replicationStreamId; + this.replicationStreamName = options.replicationStream.replicationStreamName; + this.storageConfig = options.replicationStream.getStorageConfig(); + this.factory = options.factory; + this.logger = options.replicationStream.logger; + } + + get writeCheckpointMode(): storage.WriteCheckpointMode { + return this.writeCheckpointModeValue; + } + + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { + this.writeCheckpointModeValue = mode; + } + + async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { + if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) { + throw new errors.ValidationError( + `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` + ); + } + + let createdCheckpoint: bigint | null = null; + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + const [latest] = await transactionalEntityManager.find( + this.options.dialect.writeCheckpointEntity, + { + userId: checkpoint.user_id, + syncRulesId: null + }, + { + orderBy: { checkpoint: 'DESC' }, + limit: 1 + } + ); + createdCheckpoint = (latest?.checkpoint ?? 0n) + 1n; + const row = transactionalEntityManager.create(this.options.dialect.writeCheckpointEntity, { + id: uuid.v4(), + syncRulesId: null, + userId: checkpoint.user_id, + checkpoint: createdCheckpoint, + heads: checkpoint.heads, + createdAt: new Date() + }); + + transactionalEntityManager.persist(row); + await transactionalEntityManager.flush(); + }); + + this.factory.checkpointWatcher.notify(); + return createdCheckpoint!; + } + + async lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { + switch (this.writeCheckpointMode) { + case storage.WriteCheckpointMode.CUSTOM: + return this.lastCustomWriteCheckpoint({ + user_id: filters.user_id, + sync_rules_id: this.replicationStreamId + }); + case storage.WriteCheckpointMode.MANAGED: + if (!('heads' in filters)) { + throw new errors.ValidationError(`Replication HEAD is required for managed Write Checkpoint filtering`); + } + return this.lastManagedWriteCheckpoint(filters); + } + } + + async createWriter(options: storage.CreateWriterOptions): Promise { + const em = this.options.orm.em.fork(); + const syncRules = await em.findOne(this.options.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + + const checkpointLsn = syncRules?.lastCheckpointLsn ?? null; + const writer = new MikroOrmBucketBatch({ + factory: this.factory, + orm: this.options.orm, + dialect: this.options.dialect, + logger: options.logger ?? this.logger, + syncRules: this.getParsedSyncRules(options), + replicationStreamId: this.replicationStreamId, + replicationStreamName: this.replicationStreamName, + lastCheckpointLsn: checkpointLsn, + keepaliveOp: syncRules?.keepaliveOp ?? null, + resumeFromLsn: utils.maxLsn(syncRules?.snapshotLsn, checkpointLsn), + storeCurrentData: options.storeCurrentData, + skipExistingRows: options.skipExistingRows ?? false, + markRecordUnavailable: options.markRecordUnavailable, + hooks: options.hooks + }); + this.iterateListeners((cb) => cb.batchStarted?.(writer)); + return writer; + } + + async startBatch( + options: storage.CreateWriterOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + await using writer = await this.createWriter(options); + await callback(writer); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + } + + getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig { + const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; + if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { + this.parsedSyncRulesCache = { + parsed: this.options.replicationStream.parsed(options).hydratedSyncConfig(), + options + }; + } + + return this.parsedSyncRulesCache!.parsed; + } + + async terminate(options?: storage.TerminateOptions): Promise { + if (!options || options.clearStorage) { + await this.clear(options); + } + + const em = this.options.orm.em.fork(); + const row = await em.findOne(this.options.dialect.syncRulesEntity, { id: this.replicationStreamId }); + if (row != null) { + em.assign(row, { + state: storage.SyncRuleState.TERMINATED, + snapshotDone: false + }); + await em.flush(); + } + this.factory.checkpointWatcher.notify(); + } + + async getStatus(): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOne(this.options.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + + return { + checkpoint_lsn: row?.lastCheckpointLsn ?? null, + active: row?.state == storage.SyncRuleState.ACTIVE, + snapshot_done: row?.snapshotDone ?? false, + snapshot_lsn: row?.snapshotLsn ?? null, + keepalive_op: row?.keepaliveOp ?? null + }; + } + + async clear(_options?: storage.ClearStorageOptions): Promise { + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + const row = await transactionalEntityManager.findOne(this.options.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + if (row != null) { + transactionalEntityManager.assign(row, { + snapshotDone: false, + lastCheckpointLsn: null, + lastCheckpoint: null, + noCheckpointBefore: null + }); + } + + await transactionalEntityManager.nativeDelete(this.options.dialect.bucketDataEntity, { + groupId: this.replicationStreamId + }); + await transactionalEntityManager.nativeDelete(this.options.dialect.bucketParametersEntity, { + groupId: this.replicationStreamId + }); + await transactionalEntityManager.nativeDelete(this.options.dialect.currentDataEntity, { + groupId: this.replicationStreamId + }); + await transactionalEntityManager.nativeDelete(this.options.dialect.sourceTableEntity, { + groupId: this.replicationStreamId + }); + }); + + this.factory.checkpointWatcher.notify(); + } + + async reportError(e: any): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOne(this.options.dialect.syncRulesEntity, { id: this.replicationStreamId }); + if (row != null) { + em.assign(row, { + lastFatalError: String(e.message ?? 'Replication failure'), + lastFatalErrorTs: new Date() + }); + await em.flush(); + } + } + + async compact(options?: storage.CompactOptions): Promise { + let maxOpId = options?.maxOpId; + if (maxOpId == null) { + const checkpoint = await this.getCheckpoint(); + maxOpId = checkpoint.checkpoint; + } + + const compactor = new MikroOrmCompactor(this.options.orm, this.options.dialect, this.replicationStreamId, { + ...options, + maxOpId, + logger: this.logger + }); + await compactor.compact(); + + if (options?.compactParameterData) { + await compactor.compactParameterData(options); + } + } + + async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise { + return { buckets: 0 }; + } + + async getCheckpoint(): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOne(this.options.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + + return { + checkpoint: row?.lastCheckpoint ?? 0n, + lsn: row?.lastCheckpointLsn ?? null, + getParameterSets: (lookups, limit) => this.getParameterSets(row?.lastCheckpoint ?? 0n, lookups, limit) + }; + } + + async getCheckpointChanges(_options: GetCheckpointChangesOptions): Promise { + return CHECKPOINT_INVALIDATE_ALL; + } + + async *watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable { + let lastCheckpoint: bigint | null = null; + let lastCheckpointLsn: string | null = null; + let lastWriteCheckpoint: bigint | null = null; + const { signal, user_id } = options; + + const watcher = this.factory.checkpointWatcher.watch(signal)[Symbol.asyncIterator](); + let nextNotification: Promise> | null = null; + let readImmediately = true; + + try { + while (!signal.aborted) { + if (!readImmediately) { + nextNotification ??= watcher.next(); + const result = await nextNotification; + nextNotification = null; + if (result.done) { + return; + } + } + readImmediately = false; + + if (signal.aborted) { + return; + } + + const base = await this.getCheckpoint(); + const currentWriteCheckpoint = await this.lastWriteCheckpoint({ + user_id, + heads: base.lsn == null ? {} : { '1': base.lsn } + }); + + if ( + currentWriteCheckpoint == lastWriteCheckpoint && + base.checkpoint == lastCheckpoint && + base.lsn == lastCheckpointLsn + ) { + continue; + } + + lastWriteCheckpoint = currentWriteCheckpoint; + lastCheckpoint = base.checkpoint; + lastCheckpointLsn = base.lsn; + nextNotification = watcher.next(); + + yield { + base, + writeCheckpoint: currentWriteCheckpoint, + update: CHECKPOINT_INVALIDATE_ALL + }; + } + } finally { + await watcher.return?.(); + } + } + + async *getBucketDataBatch( + checkpoint: bigint, + dataBuckets: BucketDataRequest[], + options?: BucketDataBatchOptions + ): AsyncIterable { + if (dataBuckets.length == 0) { + return; + } + + const batchRowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + const startOpByBucket = new Map(dataBuckets.map((request) => [request.bucket, request.start])); + const rows = this.options.dialect.streamBucketDataRows({ + em: this.options.orm.em.fork(), + groupId: this.replicationStreamId, + checkpoint, + dataBuckets, + limit: batchRowLimit + }); + + let chunkSizeBytes = 0; + let currentChunk: utils.SyncBucketData | null = null; + let targetOp: bigint | null = null; + let batchRowCount = 0; + + for await (const row of rows) { + const rowSizeBytes = row.data?.length ?? 0; + const sizeExceeded = + chunkSizeBytes >= chunkSizeLimitBytes || + ((currentChunk?.data.length ?? 0) > 0 && chunkSizeBytes + rowSizeBytes > chunkSizeLimitBytes) || + (currentChunk?.data.length ?? 0) >= batchRowLimit; + + if (currentChunk == null || currentChunk.bucket != row.bucketName || sizeExceeded) { + let start: string | undefined = undefined; + if (currentChunk != null) { + if (currentChunk.bucket == row.bucketName) { + currentChunk.has_more = true; + start = currentChunk.next_after; + } + + const yieldChunk = currentChunk; + currentChunk = null; + chunkSizeBytes = 0; + yield { chunkData: yieldChunk, targetOp }; + targetOp = null; + if (batchRowCount >= batchRowLimit) { + break; + } + } + + if (start == null) { + const startOpId = startOpByBucket.get(row.bucketName); + if (startOpId == null) { + throw new Error(`data for unexpected bucket: ${row.bucketName}`); + } + start = utils.internalToExternalOpId(startOpId); + } + currentChunk = { + bucket: row.bucketName, + after: start, + has_more: false, + data: [], + next_after: start + }; + } + + const entry = bucketDataRowToOpEntry(row); + currentChunk.data.push(entry); + currentChunk.next_after = entry.op_id; + if (row.targetOp != null && (targetOp == null || row.targetOp > targetOp)) { + targetOp = row.targetOp; + } + + chunkSizeBytes += rowSizeBytes; + batchRowCount++; + } + + if (currentChunk != null) { + currentChunk.has_more = batchRowCount >= batchRowLimit; + yield { chunkData: currentChunk, targetOp }; + } + } + + async getChecksums(checkpoint: bigint, buckets: BucketChecksumRequest[]): Promise { + const result: utils.ChecksumMap = new Map(); + for (const bucket of buckets) { + const rows = await this.options.orm.em.fork().find( + this.options.dialect.bucketDataEntity, + { + groupId: this.replicationStreamId, + bucketName: bucket.bucket, + opId: { $lte: checkpoint } + }, + { + orderBy: { opId: 'ASC' } + } + ); + const entries = rows.map(bucketDataRowToOpEntry); + const checksum = entries.reduce((total, entry) => utils.addChecksums(total, Number(entry.checksum)), 0); + result.set(bucket.bucket, { + bucket: bucket.bucket, + checksum, + count: entries.length + }); + } + return result; + } + + clearChecksumCache(): void { + // No checksum cache exists in the initial MikroORM storage slice. + } + + private async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters): Promise { + const row = await this.options.orm.em.fork().findOne( + this.options.dialect.writeCheckpointEntity, + { + userId: filters.user_id, + syncRulesId: filters.sync_rules_id + }, + { + orderBy: { checkpoint: 'DESC' } + } + ); + return row?.checkpoint ?? null; + } + + private async getParameterSets( + checkpoint: bigint, + lookups: sync_rules.ScopedParameterLookup[], + limit: number + ): Promise { + const resultsByLookup = new Map(); + let totalRows = 0; + + for (const lookup of lookups) { + const serializedLookup = storage.serializeLookupBuffer(lookup); + const rows = await this.options.orm.em.fork().find( + this.options.dialect.bucketParametersEntity, + { + groupId: this.replicationStreamId, + lookup: serializedLookup, + id: { $lte: checkpoint } + }, + { + orderBy: { id: 'DESC' } + } + ); + + const latestBySource = new Map(); + for (const row of rows) { + const key = `${row.sourceTable}:${Buffer.from(row.sourceKey).toString('hex')}`; + if (!latestBySource.has(key)) { + latestBySource.set(key, row); + } + } + + for (const row of latestBySource.values()) { + const parameterRows = parseBucketParameters(row.bucketParameters); + if (parameterRows.length == 0) { + continue; + } + totalRows += parameterRows.length; + if (totalRows > limit) { + throw new storage.ParameterSetLimitExceededError(limit); + } + const existing = resultsByLookup.get(lookup); + if (existing != null) { + existing.push(...parameterRows); + } else { + resultsByLookup.set(lookup, parameterRows); + } + } + } + + const results: sync_rules.ParameterLookupRows[] = []; + resultsByLookup.forEach((rows, lookup) => results.push({ lookup, rows })); + return results; + } + + private async lastManagedWriteCheckpoint(filters: storage.ManagedWriteCheckpointFilters): Promise { + const lsn = filters.heads['1']; + if (lsn == null) { + return null; + } + + const rows = await this.options.orm.em.fork().find( + this.options.dialect.writeCheckpointEntity, + { + userId: filters.user_id, + syncRulesId: null + }, + { + orderBy: { checkpoint: 'DESC' } + } + ); + + return ( + rows.find((row) => { + const rowHead = getPrimaryReplicationHead(row.heads); + return rowHead != null && rowHead <= lsn; + })?.checkpoint ?? null + ); + } +} + +function getPrimaryReplicationHead(heads: unknown): string | null { + if (heads == null || typeof heads != 'object' || Array.isArray(heads)) { + return null; + } + + const head = (heads as Record)['1']; + return typeof head == 'string' ? head : null; +} + +function parseBucketParameters(value: unknown): sync_rules.SqliteJsonRow[] { + if (typeof value == 'string') { + return JSONBig.parse(value) as sync_rules.SqliteJsonRow[]; + } + return Array.isArray(value) ? (value as sync_rules.SqliteJsonRow[]) : []; +} + +function bucketDataRowToOpEntry(row: BucketData): utils.OplogEntry { + if (row.op == 'PUT' || row.op == 'REMOVE') { + return { + op_id: utils.internalToExternalOpId(row.opId), + op: row.op, + object_type: row.tableName ?? undefined, + object_id: row.rowId ?? undefined, + checksum: Number(row.checksum), + subkey: + row.sourceTable != null && row.sourceKey != null + ? replicaIdToSubkey(row.sourceTable, storage.deserializeReplicaId(Buffer.from(row.sourceKey))) + : undefined, + data: row.op == 'REMOVE' ? null : (row.data ?? undefined) + }; + } + + return { + op_id: utils.internalToExternalOpId(row.opId), + op: row.op as 'CLEAR' | 'MOVE', + checksum: Number(row.checksum) + }; +} + +function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string { + if (storage.isUUID(id)) { + return `${tableId}/${id.toHexString()}`; + } + return uuid.v5(storage.serializeBson({ table: tableId, id }), utils.ID_NAMESPACE); +} diff --git a/modules/module-mikroorm-storage/src/storage/storage-index.ts b/modules/module-mikroorm-storage/src/storage/storage-index.ts new file mode 100644 index 000000000..fa6d1b339 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/storage-index.ts @@ -0,0 +1,8 @@ +export * from './MikroOrmBucketBatch.js'; +export * from './MikroOrmBucketStorageFactory.js'; +export * from './MikroOrmPersistedBatch.js'; +export * from './MikroOrmPersistedReplicationStream.js'; +export * from './MikroOrmReportStorage.js'; +export * from './MikroOrmStorageDialect.js'; +export * from './MikroOrmStorageProvider.js'; +export * from './MikroOrmSyncRulesStorage.js'; diff --git a/modules/module-mikroorm-storage/src/storage/unsupported.ts b/modules/module-mikroorm-storage/src/storage/unsupported.ts new file mode 100644 index 000000000..9b00ceb16 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/unsupported.ts @@ -0,0 +1,3 @@ +export function unsupportedMikroOrmStorageFeature(feature: string): never { + throw new Error(`MikroORM bucket storage does not implement ${feature} yet.`); +} diff --git a/modules/module-mikroorm-storage/src/types/types.ts b/modules/module-mikroorm-storage/src/types/types.ts new file mode 100644 index 000000000..1ac2271de --- /dev/null +++ b/modules/module-mikroorm-storage/src/types/types.ts @@ -0,0 +1,45 @@ +import { configFile } from '@powersync/service-types'; +import * as t from 'ts-codec'; + +export const MIKRO_ORM_SQLITE_STORAGE_TYPE = 'mikroorm:sqlite'; + +export const MikroOrmSqliteStorageConfig = configFile.BaseStorageConfig.and( + t.object({ + type: t.literal(MIKRO_ORM_SQLITE_STORAGE_TYPE), + /** + * SQLite database file. Use ":memory:" for in-memory storage. + */ + filename: t.string, + /** + * Optional base directory for MikroORM migrations. + */ + migrations_path: t.string.optional() + }) +); + +export type MikroOrmSqliteStorageConfig = t.Encoded; +export type MikroOrmSqliteStorageConfigDecoded = t.Decoded; + +export const isMikroOrmSqliteStorageConfig = ( + config: configFile.GenericStorageConfig +): config is MikroOrmSqliteStorageConfig => { + return config.type == MIKRO_ORM_SQLITE_STORAGE_TYPE; +}; + +export interface NormalizedMikroOrmSqliteStorageConfig { + type: typeof MIKRO_ORM_SQLITE_STORAGE_TYPE; + filename: string; + migrations_path?: string; + max_pool_size: number; +} + +export function normalizeMikroOrmSqliteStorageConfig( + config: MikroOrmSqliteStorageConfigDecoded +): NormalizedMikroOrmSqliteStorageConfig { + return { + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: config.filename, + migrations_path: config.migrations_path, + max_pool_size: config.max_pool_size ?? 1 + }; +} diff --git a/modules/module-mikroorm-storage/test/src/__snapshots__/storage.test.ts.snap b/modules/module-mikroorm-storage/test/src/__snapshots__/storage.test.ts.snap new file mode 100644 index 000000000..316577eaf --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/__snapshots__/storage.test.ts.snap @@ -0,0 +1,124 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v1 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v1 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v2 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v2 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v3 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v3 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; diff --git a/modules/module-mikroorm-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-mikroorm-storage/test/src/__snapshots__/storage_sync.test.ts.snap new file mode 100644 index 000000000..3c1a48a99 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -0,0 +1,3568 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`sync - MikroORM SQLite > storage v1 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; diff --git a/modules/module-mikroorm-storage/test/src/migrations.test.ts b/modules/module-mikroorm-storage/test/src/migrations.test.ts new file mode 100644 index 000000000..1ce98fca3 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/migrations.test.ts @@ -0,0 +1,174 @@ +import { rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { MikroORM } from '@mikro-orm/core'; +import { SqliteDriver } from '@mikro-orm/sqlite'; +import { Direction } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createSqliteMikroOrmOptions } from '../../src/drivers/sqlite/sqlite-config.js'; +import { sqliteMikroOrmStorageDialect } from '../../src/drivers/sqlite/sqlite-dialect.js'; +import { MIKRO_ORM_SQLITE_STORAGE_TYPE } from '../../src/index.js'; +import { MikroOrmMigrationAgent } from '../../src/migrations/MikroOrmMigrationAgent.js'; +import { normalizeMikroOrmSqliteStorageConfig } from '../../src/types/types.js'; + +describe('MikroORM migrations', () => { + const dbFiles: string[] = []; + + const createDbFile = () => { + const filename = join(tmpdir(), `powersync-mikroorm-storage-${process.pid}-${Date.now()}-${dbFiles.length}.sqlite`); + dbFiles.push(filename); + return filename; + }; + + afterEach(async () => { + await Promise.all(dbFiles.splice(0).map((file) => rm(file, { force: true }))); + }); + + it('runs SQLite migrations through the service migration agent', async () => { + const filename = createDbFile(); + + await using agent = new MikroOrmMigrationAgent({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename + }); + + await agent.run({ + direction: Direction.Up, + migrations: [] + }); + + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename + }) + ) + ); + + try { + const tables = await orm.em + .getConnection() + .execute<{ name: string }[]>(`select name from sqlite_master where type = 'table' order by name`); + const tableNames = tables.map((row) => row.name); + + expect(tableNames).toContain('powersync_migration_locks'); + expect(tableNames).toContain('mikro_orm_migrations'); + expect(tableNames).toContain('instance'); + expect(tableNames).toContain('sync_rules'); + expect(tableNames).toContain('bucket_data'); + expect(tableNames).toContain('write_checkpoints'); + + const indexes = await orm.em + .getConnection() + .execute<{ name: string }[]>(`select name from sqlite_master where type = 'index' order by name`); + const indexNames = indexes.map((row) => row.name); + + expect(indexNames).toContain('bucket_data_bucket_op_index'); + expect(indexNames).toContain('bucket_parameters_lookup_index'); + expect(indexNames).toContain('bucket_parameters_source_index'); + expect(indexNames).toContain('current_data_pending_delete_index'); + expect(indexNames).toContain('source_table_lookup'); + expect(indexNames).toContain('write_checkpoints_user_checkpoint_index'); + } finally { + await orm.close(true); + } + }); + + it('hydrates SQLite-specific storage columns as service-facing values', async () => { + const filename = createDbFile(); + + await using agent = new MikroOrmMigrationAgent({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename + }); + + await agent.run({ + direction: Direction.Up, + migrations: [] + }); + + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename + }) + ) + ); + + try { + const em = orm.em.fork(); + const bucketRow = em.create(sqliteMikroOrmStorageDialect.bucketDataEntity, { + id: 'test-bucket-row', + groupId: 1, + bucketName: 'bucket[]', + opId: 42n, + op: 'PUT', + sourceTable: null, + sourceKey: Buffer.from('source-key'), + tableName: null, + rowId: null, + checksum: 99n, + data: null, + targetOp: null + }); + + const syncPlan: storage.SerializedSyncPlan = { + plan: { + version: 1, + dataSources: [], + buckets: [], + parameterIndexes: [], + streams: [] + }, + compatibility: { + edition: 1, + overrides: {} + }, + eventDescriptors: {} + }; + + const syncRulesRow = em.create(sqliteMikroOrmStorageDialect.syncRulesEntity, { + state: storage.SyncRuleState.PROCESSING, + snapshotDone: false, + snapshotLsn: null, + lastCheckpoint: 123n, + lastCheckpointLsn: null, + noCheckpointBefore: null, + slotName: 'test_slot', + lastCheckpointTs: null, + lastKeepaliveTs: null, + lastFatalError: null, + lastFatalErrorTs: null, + keepaliveOp: 124n, + storageVersion: storage.CURRENT_STORAGE_VERSION, + content: 'bucket_definitions: []', + syncPlan + }); + + em.persist([bucketRow, syncRulesRow]); + await em.flush(); + em.clear(); + + const storedBucketRow = await em.findOneOrFail(sqliteMikroOrmStorageDialect.bucketDataEntity, { + id: 'test-bucket-row' + }); + const storedSyncRulesRow = await em.findOneOrFail(sqliteMikroOrmStorageDialect.syncRulesEntity, { + id: syncRulesRow.id + }); + + expect(storedBucketRow.opId).toBe(42n); + expect(storedBucketRow.checksum).toBe(99n); + expect(storedBucketRow.sourceKey).toBeInstanceOf(Buffer); + expect(storedBucketRow.sourceKey?.toString()).toBe('source-key'); + expect(storedSyncRulesRow.lastCheckpoint).toBe(123n); + expect(storedSyncRulesRow.keepaliveOp).toBe(124n); + expect(storedSyncRulesRow.syncPlan).toEqual(syncPlan); + } finally { + await orm.close(true); + } + }); +}); diff --git a/modules/module-mikroorm-storage/test/src/setup.ts b/modules/module-mikroorm-storage/test/src/setup.ts new file mode 100644 index 000000000..b14ebcec9 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/setup.ts @@ -0,0 +1,11 @@ +import { container } from '@powersync/lib-services-framework'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; +import { beforeAll, beforeEach } from 'vitest'; + +beforeAll(async () => { + container.registerDefaults(); +}); + +beforeEach(async () => { + METRICS_HELPER.resetMetrics(); +}); diff --git a/modules/module-mikroorm-storage/test/src/storage-provider.test.ts b/modules/module-mikroorm-storage/test/src/storage-provider.test.ts new file mode 100644 index 000000000..db21c4c17 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/storage-provider.test.ts @@ -0,0 +1,38 @@ +import { storage, system } from '@powersync/service-core'; +import { describe, expect, it } from 'vitest'; +import { MIKRO_ORM_SQLITE_STORAGE_TYPE, MikroOrmStorageProvider } from '../../src/index.js'; + +const RESOLVED_CONFIG = { + storage: { + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: ':memory:' + }, + slot_name_prefix: 'test_' +} as const; + +function getStorageOptions(serviceMode: system.ServiceContextMode): storage.GetStorageOptions { + return { + resolvedConfig: RESOLVED_CONFIG as unknown as storage.GetStorageOptions['resolvedConfig'], + serviceMode + }; +} + +describe('MikroORM SQLite storage provider', () => { + it('rejects split service runners', async () => { + const provider = new MikroOrmStorageProvider(); + + await expect(provider.getStorage(getStorageOptions(system.ServiceContextMode.API))).rejects.toThrow( + 'MikroORM SQLite storage only supports the unified service runner' + ); + await expect(provider.getStorage(getStorageOptions(system.ServiceContextMode.SYNC))).rejects.toThrow( + 'MikroORM SQLite storage only supports the unified service runner' + ); + }); + + it('allows the unified service runner', async () => { + const provider = new MikroOrmStorageProvider(); + const activeStorage = await provider.getStorage(getStorageOptions(system.ServiceContextMode.UNIFIED)); + + await activeStorage.shutDown(); + }); +}); diff --git a/modules/module-mikroorm-storage/test/src/storage.test.ts b/modules/module-mikroorm-storage/test/src/storage.test.ts new file mode 100644 index 000000000..d9e147c01 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/storage.test.ts @@ -0,0 +1,21 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { MIKRO_ORM_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe('Sync Bucket Validation', register.registerBucketValidationTests); + +for (let storageVersion of TEST_STORAGE_VERSIONS) { + describe(`MikroORM SQLite Sync Bucket Storage - Parameters - v${storageVersion}`, () => + register.registerDataStorageParameterTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`MikroORM SQLite Sync Bucket Storage - Data - v${storageVersion}`, () => + register.registerDataStorageDataTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`MikroORM SQLite Sync Bucket Storage - Checkpoints - v${storageVersion}`, () => + register.registerDataStorageCheckpointTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`MikroORM SQLite Sync Bucket Storage - Compaction - v${storageVersion}`, () => { + register.registerCompactTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion }); + register.registerParameterCompactTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion }); + }); +} diff --git a/modules/module-mikroorm-storage/test/src/storage_sync.test.ts b/modules/module-mikroorm-storage/test/src/storage_sync.test.ts new file mode 100644 index 000000000..ab6ff1760 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/storage_sync.test.ts @@ -0,0 +1,14 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { MIKRO_ORM_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe('sync - MikroORM SQLite', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`storage v${storageVersion}`, () => { + register.registerSyncTests(MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory, { + storageVersion, + tableIdStrings: MIKRO_ORM_SQLITE_STORAGE_FACTORY.tableIdStrings + }); + }); + } +}); diff --git a/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts b/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts new file mode 100644 index 000000000..45e65dc8f --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts @@ -0,0 +1,99 @@ +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { describe, expect, it } from 'vitest'; +import type { MikroOrmBucketStorageFactory } from '../../src/index.js'; +import { MIKRO_ORM_SQLITE_STORAGE_FACTORY } from './util.js'; + +describe('MikroORM SyncRules storage', () => { + const syncRules = updateSyncRulesFromYaml( + ` +bucket_definitions: + mybucket: + data: [] +`, + { + validate: false + } + ); + + it('stores and resolves managed write checkpoints', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + + const first = await bucketStorage.createManagedWriteCheckpoint({ + user_id: 'user1', + heads: { '1': '5/0' } + }); + const second = await bucketStorage.createManagedWriteCheckpoint({ + user_id: 'user1', + heads: { '1': '6/0' } + }); + + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '4/0' } })).resolves.toBeNull(); + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '5/0' } })).resolves.toBe(first); + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '6/0' } })).resolves.toBe(second); + }); + + it('watches checkpoint and managed write checkpoint changes', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + const abortController = new AbortController(); + + try { + const iterator = bucketStorage + .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) + [Symbol.asyncIterator](); + + const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({ + user_id: 'user1', + heads: { '1': '5/0' } + }); + + const em = factory.orm.em.fork(); + const row = await em.findOneOrFail(factory.dialect.syncRulesEntity, { + id: stream.replicationStreamId + }); + em.assign(row, { + lastCheckpoint: 0n, + lastCheckpointLsn: '5/0' + }); + await em.flush(); + factory.checkpointWatcher.notify(); + + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { + base: { + checkpoint: 0n, + lsn: '5/0' + }, + writeCheckpoint + } + }); + } finally { + abortController.abort(); + } + }); + + it('resolves custom write checkpoints from the write checkpoint entity', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); + + const em = factory.orm.em.fork(); + const row = em.create(factory.dialect.writeCheckpointEntity, { + id: 'custom-user1', + syncRulesId: stream.replicationStreamId, + userId: 'user1', + checkpoint: 42n, + heads: null, + createdAt: new Date() + }); + em.persist(row); + await em.flush(); + + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1' })).resolves.toBe(42n); + }); +}); diff --git a/modules/module-mikroorm-storage/test/src/util.ts b/modules/module-mikroorm-storage/test/src/util.ts new file mode 100644 index 000000000..f04bb025d --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/util.ts @@ -0,0 +1,26 @@ +import { storage, SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core'; +import { + createSqliteMikroOrmStorageFactory, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + normalizeMikroOrmSqliteStorageConfig +} from '../../src/index.js'; + +const BASE_CONFIG = { + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: ':memory:' +} as const; + +export const MIKRO_ORM_SQLITE_STORAGE_FACTORY: storage.TestStorageConfig = { + tableIdStrings: true, + factory: async () => { + const config = normalizeMikroOrmSqliteStorageConfig(BASE_CONFIG); + const factory = await createSqliteMikroOrmStorageFactory({ + config, + slotNamePrefix: 'test_' + }); + await factory.orm.schema.update(); + return factory; + } +}; + +export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; diff --git a/modules/module-mikroorm-storage/test/tsconfig.json b/modules/module-mikroorm-storage/test/tsconfig.json new file mode 100644 index 000000000..5bced458d --- /dev/null +++ b/modules/module-mikroorm-storage/test/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../tsconfig.tests.json", + "compilerOptions": { + "declarationDir": "dist/@types", + "tsBuildInfoFile": "dist/.tsbuildinfo", + "lib": ["ES2022", "esnext.disposable"], + "rootDir": "src" + }, + "include": ["src"], + "references": [ + { + "path": "../" + }, + { + "path": "../../../packages/service-core-tests" + } + ] +} diff --git a/modules/module-mikroorm-storage/tsconfig.json b/modules/module-mikroorm-storage/tsconfig.json new file mode 100644 index 000000000..048854242 --- /dev/null +++ b/modules/module-mikroorm-storage/tsconfig.json @@ -0,0 +1,25 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [ + { + "path": "../../packages/types" + }, + { + "path": "../../packages/sync-rules" + }, + { + "path": "../../packages/service-core" + }, + { + "path": "../../libs/lib-services" + } + ] +} diff --git a/modules/module-mikroorm-storage/vitest.config.ts b/modules/module-mikroorm-storage/vitest.config.ts new file mode 100644 index 000000000..285c29477 --- /dev/null +++ b/modules/module-mikroorm-storage/vitest.config.ts @@ -0,0 +1,7 @@ +import { serviceIntegrationTestConfig } from '../test_config'; + +const config = serviceIntegrationTestConfig(__dirname); +config.test ??= {}; +config.test.testTimeout = 30_000; + +export default config; diff --git a/packages/schema/package.json b/packages/schema/package.json index edd35c3e5..84e9d0fb8 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -32,6 +32,7 @@ "@powersync/service-module-postgres-storage": "workspace:*", "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-mikroorm-storage": "workspace:*", "@powersync/service-module-convex": "workspace:*", "@powersync/service-module-mysql": "workspace:*", "@powersync/service-module-mssql": "workspace:*", diff --git a/packages/schema/src/scripts/compile-json-schema.ts b/packages/schema/src/scripts/compile-json-schema.ts index 9d68b053b..e5702c981 100644 --- a/packages/schema/src/scripts/compile-json-schema.ts +++ b/packages/schema/src/scripts/compile-json-schema.ts @@ -1,4 +1,5 @@ import { ConvexConnectionConfig } from '@powersync/service-module-convex/types'; +import { MikroOrmSqliteStorageConfig } from '@powersync/service-module-mikroorm-storage/types'; import { MongoStorageConfig } from '@powersync/service-module-mongodb-storage/types'; import { MongoConnectionConfig } from '@powersync/service-module-mongodb/types'; import { MSSQLConnectionConfig } from '@powersync/service-module-mssql/types'; @@ -25,7 +26,9 @@ const mergedDataSourceConfig = configFile.genericDataSourceConfig .or(PostgresConnectionConfig) .or(ConvexConnectionConfig); -const mergedStorageConfig = configFile.GenericStorageConfig.or(PostgresStorageConfig).or(MongoStorageConfig); +const mergedStorageConfig = configFile.GenericStorageConfig.or(PostgresStorageConfig) + .or(MongoStorageConfig) + .or(MikroOrmSqliteStorageConfig); const mergedConfig = t.object({ ...baseShape, diff --git a/packages/service-core/src/storage/StorageEngine.ts b/packages/service-core/src/storage/StorageEngine.ts index a53819ccb..b7ddfec4d 100644 --- a/packages/service-core/src/storage/StorageEngine.ts +++ b/packages/service-core/src/storage/StorageEngine.ts @@ -5,6 +5,7 @@ import { ActiveStorage, StorageProvider } from './StorageProvider.js'; export type StorageEngineOptions = { configuration: ResolvedPowerSyncConfig; + serviceMode: string; }; export interface StorageEngineListener { @@ -45,7 +46,8 @@ export class StorageEngine extends BaseObserver { logger.info('Starting Storage Engine...'); const { configuration } = this.options; this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({ - resolvedConfig: configuration + resolvedConfig: configuration, + serviceMode: this.options.serviceMode }); this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage)); this.currentActiveStorage.onFatalError?.((error) => { diff --git a/packages/service-core/src/storage/StorageProvider.ts b/packages/service-core/src/storage/StorageProvider.ts index f2404bdf0..ed1ddad7d 100644 --- a/packages/service-core/src/storage/StorageProvider.ts +++ b/packages/service-core/src/storage/StorageProvider.ts @@ -19,6 +19,7 @@ export interface ActiveStorage { export interface GetStorageOptions { // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage resolvedConfig: util.ResolvedPowerSyncConfig; + serviceMode: string; } /** diff --git a/packages/service-core/src/system/ServiceContext.ts b/packages/service-core/src/system/ServiceContext.ts index a81528717..102a527c3 100644 --- a/packages/service-core/src/system/ServiceContext.ts +++ b/packages/service-core/src/system/ServiceContext.ts @@ -60,7 +60,8 @@ export class ServiceContextContainer implements ServiceContext { this.lifeCycleEngine = new LifeCycledSystem(); this.storageEngine = new storage.StorageEngine({ - configuration + configuration, + serviceMode: this.serviceMode }); this.storageEngine.registerListener({ storageFatalError: (error) => { diff --git a/plans/mikroorm-bucket-storage.md b/plans/mikroorm-bucket-storage.md new file mode 100644 index 000000000..6537e8660 --- /dev/null +++ b/plans/mikroorm-bucket-storage.md @@ -0,0 +1,291 @@ +# MikroORM Bucket Storage Module Plan + +## Goal + +Build a bucket storage module backed by MikroORM, with SQLite as the first supported driver and a path for future SQL drivers. The module should use common storage logic and common entity definitions wherever possible, while isolating driver-specific concerns behind a small dialect interface. + +The current first target is `mikroorm:sqlite`. + +## Lessons Learned + +- Start from the shared PowerSync storage contract, not from the database schema alone. The quickest validation came from importing the standard core storage and sync tests into the new module. +- MikroORM already handles the primitive types we need well enough for the SQLite first pass. We do not need custom bigint/blob/json property builders for the initial implementation. +- Do not redeclare entities per driver until the driver needs a real difference. Common `defineEntity` schema constants and common classes are enough for SQLite today. +- Driver-specific classes are still useful in a typical MikroORM project when a concrete entity needs methods, hooks, or driver-only behavior. For this module, keep that option open, but do not create empty SQLite subclasses just for ceremony. +- The dialect abstraction should expose concrete entity classes directly, not a nested `MikroOrmStorageEntities` registry. The extra registry layer did not buy much once the common entity classes became stable. +- Advanced read paths should stream. This matters for bucket reads and compaction, where loading all rows into memory is the wrong shape. +- Batch flushes must be split into smaller persisted chunks. A long-lived `BucketBatch` should not keep every record in one MikroORM unit of work. +- SQLite checkpoint notification is process-local only. The provider should reject split service modes for SQLite and require the unified runner, while future drivers can expose cross-process notification. +- Migration locking is a chicken-and-egg problem. The lock must be created before generated MikroORM migrations run, so the lock manager needs a tiny amount of driver-specific raw SQL. + +## Module Layout + +```text +modules/module-mikroorm-storage/ + package.json + tsconfig.json + vitest.config.ts + src/ + index.ts + mikro-orm.config.ts + module/ + MikroOrmStorageModule.ts + types/ + types.ts + entities/ + entities-index.ts + common/ + bucket-data.schema.ts + bucket-parameters.schema.ts + current-data.schema.ts + instance.schema.ts + source-table.schema.ts + sync-rules.schema.ts + write-checkpoint.schema.ts + drivers/ + sqlite/ + SqliteMikroOrmStorageFactory.ts + SqliteMigrationLockManager.ts + sqlite-config.ts + sqlite-dialect.ts + migrations/ + AbstractMikroOrmMigrationLockManager.ts + MikroOrmMigrationAgent.ts + NoOpMigrationStore.ts + sqlite/ + Migration*_InitialSqliteStorage.ts + storage/ + MikroOrmBucketBatch.ts + MikroOrmPersistedBatch.ts + MikroOrmBucketStorageFactory.ts + MikroOrmCompactor.ts + MikroOrmPersistedReplicationStream.ts + MikroOrmReportStorage.ts + MikroOrmStorageDialect.ts + MikroOrmStorageProvider.ts + MikroOrmSyncRulesStorage.ts + storage-index.ts + unsupported.ts + test/ + src/ + migrations.test.ts + storage.test.ts + storage-provider.test.ts + storage_sync.test.ts + sync-rules-storage.test.ts + util.ts +``` + +## Entity Strategy + +Use common MikroORM `defineEntity` schema constants plus common TypeScript classes: + +```ts +export class BucketData { + declare id: string; + declare groupId: number; + declare bucketName: string; + declare opId: bigint; +} + +export const BucketDataSchema = defineEntity({ + name: 'BucketData', + tableName: 'bucket_data', + properties: { + id: p.string().primary(), + groupId: p.integer().fieldName('group_id'), + bucketName: p.string().fieldName('bucket_name'), + opId: p.bigint('bigint') + } +}); +BucketDataSchema.setClass(BucketData); +``` + +The common entities currently cover: + +- `bucket_data` +- `bucket_parameters` +- `current_data` +- `instance` +- `source_tables` +- `sync_rules` +- `write_checkpoints` + +Keep indexes in the common schema when they are part of the logical access pattern and valid for SQLite. Add driver-specific indexes only when a later driver needs a different shape. + +Avoid schema factories like `defineBucketDataEntity(options)` and avoid `property-builders` for now. They made sense when we thought every driver would need custom bigint/blob/json storage, but MikroORM's built-in `p.bigint('bigint')`, `p.blob()`, `p.json()`, `p.string()`, and date handling are enough for the current module. + +Future drivers can still introduce concrete subclasses using the documented MikroORM pattern: + +```ts +export class PostgresBucketData extends BucketDataSchema.class { + // driver-specific methods or hooks +} +BucketDataSchema.setClass(PostgresBucketData); +``` + +Only do that when the subclass has a real purpose. + +## Dialect Strategy + +Storage code should depend on `MikroOrmStorageDialect`, not on SQLite imports. The dialect should expose the small set of driver-specific things the common layer needs: + +```ts +export interface MikroOrmStorageDialect { + readonly type: string; + readonly entityClasses: EntityClass[]; + readonly bucketDataEntity: EntityClass; + readonly bucketParametersEntity: EntityClass; + readonly currentDataEntity: EntityClass; + readonly instanceEntity: EntityClass; + readonly sourceTableEntity: EntityClass; + readonly syncRulesEntity: EntityClass; + readonly writeCheckpointEntity: EntityClass; + streamBucketDataRows(options: MikroOrmBucketDataStreamOptions): AsyncIterable; + createCheckpointWatcher(): MikroOrmCheckpointWatcher; +} +``` + +The dialect owns: + +- the public storage identifier, such as `mikroorm:sqlite` +- the entity classes passed to MikroORM and migration tooling +- streaming query implementation for hot bucket reads +- checkpoint watch/notify implementation +- future raw SQL escape hatches where an operation cannot be expressed well through typed ORM calls + +Keep common storage classes responsible for the high-level algorithm. Pull database-specific behavior down into dialect methods only when the common implementation would otherwise need driver-specific SQL or notification behavior. + +## SQLite First Pass + +SQLite-specific choices: + +- Use `type: 'mikroorm:sqlite'` as the public storage identifier. +- Use a `db_filename` config value, with `:memory:` supported for fast tests. +- Use MikroORM v7 and its current driver stack. Do not introduce Knex-based helpers. +- Use MikroORM's query builder streaming for bucket reads. +- Use an in-process checkpoint watcher for local wakeups. +- Reject split `api`/`sync` service runners for SQLite storage. SQLite cannot notify sharded pods, so it should only run in unified mode, while command-style contexts used by tooling remain allowed. +- Generate SQLite migrations with the MikroORM CLI instead of hand-writing schema migrations. +- Include indexes needed by the read, write, current-data, source-table, and checkpoint paths in the generated migration. + +## Migrations + +Migrations should use MikroORM migrations internally and be exposed through the standard PowerSync service migration surface. + +Implementation shape: + +- `MikroOrmMigrationAgent` extends the service migration agent and overrides `run()`. +- The service migration layer triggers the agent, but MikroORM owns migration discovery and migration state. +- `NoOpMigrationStore` exists because PowerSync should not duplicate MikroORM migration bookkeeping. +- `AbstractMikroOrmMigrationLockManager` defines the DB-backed lock contract. +- `SqliteMigrationLockManager` bootstraps its own lock table with raw SQLite. + +The raw SQLite lock bootstrap is intentional. We need a DB-backed lock before migrations run, but migrations are what normally create tables. For SQLite, the lock manager solves that classic chicken-and-egg problem with `CREATE TABLE IF NOT EXISTS` against MikroORM's connection before invoking the migrator. + +Tests should verify that the migration agent creates representative storage tables, not just that the migrator returns successfully. + +## Storage Implementation + +Implement the common storage classes in this order: + +1. `MikroOrmStorageProvider` +2. `MikroOrmBucketStorageFactory` +3. `MikroOrmPersistedReplicationStream` +4. `MikroOrmSyncRulesStorage` +5. `MikroOrmBucketBatch` +6. `MikroOrmPersistedBatch` +7. `MikroOrmCompactor` +8. `MikroOrmReportStorage` + +`MikroOrmBucketBatch` should own the public writer lifecycle: + +- listener notifications +- `resolveTables` +- `save` +- `flush` +- `truncate` +- `drop` +- checkpoint commits +- snapshot state +- write checkpoints + +`MikroOrmPersistedBatch` should own per-transaction write state: + +- preload `current_data` for a chunk +- evaluate bucket data and parameter rows +- insert `bucket_data` +- insert `bucket_parameters` +- upsert or mark-delete `current_data` + +Flush pending operations in chunks, currently `2_000` operations per persisted transaction. This keeps high-volume sync tests fast and avoids growing a single MikroORM unit of work across tens of thousands of rows. + +## Reads, Streaming, and Compaction + +Bucket reads should stream rows from the dialect through `streamBucketDataRows()`. The common sync path should consume the async iterable and batch output without buffering the entire query result. + +Compaction should also stream candidate rows. Use typed ORM calls when they express the operation cleanly. Use raw SQL only for database-specific operations that are awkward or inefficient through MikroORM, and isolate those operations behind common methods or dialect-level helpers. + +The compactor should be validated with the shared sync tests, especially checkpoint invalidation cases. + +## Test Strategy + +The module should import and run the shared core tests instead of building only bespoke SQLite tests. + +Current expected coverage: + +- shared storage tests +- shared sync tests across storage protocol versions +- migration agent tests +- sync-rules storage tests +- provider/service-mode tests + +Known verification commands: + +```sh +source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage build +source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage build:tests +source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage test test/src/storage_sync.test.ts --run +source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage test --run +``` + +Current observed status after implementing the SQLite vertical slice: + +- source build passes +- test build passes +- sync suite passes with `51 passed` +- full module suite passes with `223 passed | 2 skipped` + +There has been an intermittent native SQLite/Vitest `139` exit during full-suite startup. Rerunning has passed cleanly, so treat this as runner instability unless a failure includes test-level assertions. + +## Build Order From Scratch + +1. Create the package, TypeScript config, Vitest config, exports, and workspace wiring. +2. Add the common entity schemas and classes. +3. Add SQLite config and dialect with entity class registration. +4. Configure MikroORM migrations and generate the SQLite initial migration with the MikroORM CLI. +5. Add the DB-backed migration lock and migration agent override. +6. Add provider config for `mikroorm:sqlite` and register it with the service storage engine. +7. Implement factory and sync-rule persistence. +8. Implement writer creation and `MikroOrmBucketBatch`. +9. Split persisted operation writes into `MikroOrmPersistedBatch`. +10. Implement streamed bucket reads through the dialect. +11. Implement write checkpoints and checkpoint watching. +12. Implement compaction with streaming. +13. Add the SQLite unified-runner guard. +14. Import the shared storage and sync tests, then fill gaps until they pass. +15. Run focused sync tests and the full module suite. + +## Future Driver Guidance + +For Postgres or another SQL driver: + +- reuse the common entity definitions first +- add concrete entity subclasses only when methods/hooks/driver behavior require them +- add a new dialect object with its own entity class list +- implement cross-process checkpoint notification in `createCheckpointWatcher()` +- move raw SQL into dialect methods when query shape differs by database +- keep migrations generated by MikroORM for that driver +- add driver-specific migration locking before invoking the migrator + +This keeps the module abstract without over-abstracting the entity model before a second driver proves what actually varies. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4c2d7ad03..368ec7234 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -263,6 +263,52 @@ importers: specifier: ^5.8.5 version: 5.8.5 + modules/module-mikroorm-storage: + dependencies: + '@mikro-orm/core': + specifier: ^7.1.4 + version: 7.1.4 + '@mikro-orm/migrations': + specifier: ^7.1.4 + version: 7.1.4(@mikro-orm/core@7.1.4) + '@mikro-orm/sql': + specifier: ^7.1.4 + version: 7.1.4(@mikro-orm/core@7.1.4) + '@mikro-orm/sqlite': + specifier: ^7.1.4 + version: 7.1.4(@mikro-orm/core@7.1.4) + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../../libs/lib-services + '@powersync/service-core': + specifier: workspace:* + version: link:../../packages/service-core + '@powersync/service-jsonbig': + specifier: workspace:* + version: link:../../packages/jsonbig + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + uuid: + specifier: 'catalog:' + version: 14.0.0 + devDependencies: + '@mikro-orm/cli': + specifier: ^7.1.4 + version: 7.1.4 + '@powersync/service-core-tests': + specifier: workspace:* + version: link:../../packages/service-core-tests + typescript: + specifier: 'catalog:' + version: 6.0.3 + modules/module-mongodb: dependencies: '@powersync/lib-service-mongodb': @@ -603,6 +649,9 @@ importers: '@powersync/service-module-convex': specifier: workspace:* version: link:../../modules/module-convex + '@powersync/service-module-mikroorm-storage': + specifier: workspace:* + version: link:../../modules/module-mikroorm-storage '@powersync/service-module-mongodb': specifier: workspace:* version: link:../../modules/module-mongodb @@ -827,6 +876,9 @@ importers: '@powersync/service-module-core': specifier: workspace:* version: link:../modules/module-core + '@powersync/service-module-mikroorm-storage': + specifier: workspace:* + version: link:../modules/module-mikroorm-storage '@powersync/service-module-mongodb': specifier: workspace:* version: link:../modules/module-mongodb @@ -1341,6 +1393,38 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} + '@mikro-orm/cli@7.1.4': + resolution: {integrity: sha512-o1AQgwVJ3i8fGpR+48bSTum/Ar5u38WfyX1HykoPR03M5IdOHJ2QdiwA0o58TsA5xd0U+IlhlNLJEwZ8XCNXGA==} + engines: {node: '>= 22.17.0'} + hasBin: true + + '@mikro-orm/core@7.1.4': + resolution: {integrity: sha512-JPpOoBw2r98hVmsa7kFdt9HDqTa6/x9dhLL2MZRJQt1HwNg7S7A6Eh+nJ21dmB5QfXhu2PXqt03Y+xZZDnUD0w==} + engines: {node: '>= 22.17.0'} + peerDependencies: + dataloader: 2.2.3 + peerDependenciesMeta: + dataloader: + optional: true + + '@mikro-orm/migrations@7.1.4': + resolution: {integrity: sha512-M9rsHpuuaWavoXb0yitgwb+Dy2wzIEzFh5cdJ1NHjPVwyWrTiTTcKmZUiytvHInzLvusuTomRKNRiNUENqnFvQ==} + engines: {node: '>= 22.17.0'} + peerDependencies: + '@mikro-orm/core': 7.1.4 + + '@mikro-orm/sql@7.1.4': + resolution: {integrity: sha512-YOie3z0Y24XlzWPUrATGJja5U6Ij81gji9yEDT6JkZxJH9uF1tbmZ2qX9WHectnMB5h8JHCic5AMvaflhBviNg==} + engines: {node: '>= 22.17.0'} + peerDependencies: + '@mikro-orm/core': 7.1.4 + + '@mikro-orm/sqlite@7.1.4': + resolution: {integrity: sha512-0Buc7GMkib22SliR4dj4O3Yzs26k8lzRn5xRRpgkLpt+Mlse9ac3Mk+2ilLTwsjHm2yoIIRQbfNnhVk1z0UoSw==} + engines: {node: '>= 22.17.0'} + peerDependencies: + '@mikro-orm/core': 7.1.4 + '@mongodb-js/saslprep@1.3.1': resolution: {integrity: sha512-6nZrq5kfAz0POWyhljnbWQQJQ5uT8oE2ddX303q1uY0tWsivWKgBDXBBvuFPwOqRRalXJuVO9EjOdVtuhLX0zg==} @@ -2325,6 +2409,10 @@ packages: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} + better-sqlite3@12.10.0: + resolution: {integrity: sha512-CyzaZRQKyHkB2ZInfTTl2nvT33EbDpjkLEbE8/Zck3Ll6O0qqvuGdrJ45HgtH+HykRg88ITY3AdreBGN70aBSQ==} + engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x || 26.x} + big-integer@1.6.51: resolution: {integrity: sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==} engines: {node: '>=0.6'} @@ -2340,6 +2428,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -2721,6 +2812,9 @@ packages: resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} engines: {node: '>=18'} + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -3046,6 +3140,10 @@ packages: kuler@2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} + kysely@0.29.2: + resolution: {integrity: sha512-s6WVJyEZrbm6jhBpiKHsGHyePMrVQKJ85wZCFCr9W4QHv6WTjWIrdvTmO9hDEA3bNK0xkrE2DqrHsXMLWuZpQg==} + engines: {node: '>=22.0.0'} + leven@3.1.0: resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} engines: {node: '>=6'} @@ -3222,6 +3320,10 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + mikro-orm@7.1.4: + resolution: {integrity: sha512-6e71y7tgSWNHMPCsTBxAaEaeBKly99fCYIOByrUVTAP/lm/N9iHYVEATeC7/oqVDYINDzdlJGKk9RI8objPl4A==} + engines: {node: '>= 22.17.0'} + mimic-fn@2.1.0: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} @@ -4863,6 +4965,33 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 + '@mikro-orm/cli@7.1.4': + dependencies: + '@mikro-orm/core': 7.1.4 + mikro-orm: 7.1.4 + yargs: 17.7.2 + transitivePeerDependencies: + - dataloader + + '@mikro-orm/core@7.1.4': {} + + '@mikro-orm/migrations@7.1.4(@mikro-orm/core@7.1.4)': + dependencies: + '@mikro-orm/core': 7.1.4 + '@mikro-orm/sql': 7.1.4(@mikro-orm/core@7.1.4) + + '@mikro-orm/sql@7.1.4(@mikro-orm/core@7.1.4)': + dependencies: + '@mikro-orm/core': 7.1.4 + kysely: 0.29.2 + + '@mikro-orm/sqlite@7.1.4(@mikro-orm/core@7.1.4)': + dependencies: + '@mikro-orm/core': 7.1.4 + '@mikro-orm/sql': 7.1.4(@mikro-orm/core@7.1.4) + better-sqlite3: 12.10.0 + kysely: 0.29.2 + '@mongodb-js/saslprep@1.3.1': dependencies: sparse-bitfield: 3.0.3 @@ -5905,6 +6034,11 @@ snapshots: dependencies: is-windows: 1.0.2 + better-sqlite3@12.10.0: + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.3 + big-integer@1.6.51: {} big-integer@1.6.52: {} @@ -5913,6 +6047,10 @@ snapshots: binary-extensions@2.3.0: {} + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + bl@4.1.0: dependencies: buffer: 5.7.1 @@ -6292,6 +6430,8 @@ snapshots: dependencies: is-unicode-supported: 2.1.0 + file-uri-to-path@1.0.0: {} + fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -6589,6 +6729,8 @@ snapshots: kuler@2.0.0: {} + kysely@0.29.2: {} + leven@3.1.0: {} light-my-request@6.6.0: @@ -6730,6 +6872,8 @@ snapshots: braces: 3.0.3 picomatch: 2.3.2 + mikro-orm@7.1.4: {} + mimic-fn@2.1.0: {} mimic-function@5.0.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7cba9ccfc..68489af56 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -19,9 +19,11 @@ allowBuilds: # These need install scripts to download/build native modules '@mongodb-js/zstd': true esbuild: false + sqlite3: true # We don't need to run these checks: https://github.com/protobufjs/protobuf.js/blob/master/scripts/postinstall.js protobufjs: false + better-sqlite3: true overrides: # Override to remove vitest production dependency diff --git a/service/package.json b/service/package.json index b70419d2f..7fe28c327 100644 --- a/service/package.json +++ b/service/package.json @@ -16,6 +16,7 @@ "@powersync/service-module-postgres-storage": "workspace:*", "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-mikroorm-storage": "workspace:*", "@powersync/service-module-mssql": "workspace:*", "@powersync/service-module-mysql": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", diff --git a/service/src/util/modules.ts b/service/src/util/modules.ts index 45631cd7e..5232dc05f 100644 --- a/service/src/util/modules.ts +++ b/service/src/util/modules.ts @@ -11,6 +11,8 @@ export const DYNAMIC_MODULES: core.ModuleLoaders = { storage: { mongodb: () => import('@powersync/service-module-mongodb-storage').then((module) => new module.MongoStorageModule()), + 'mikroorm:sqlite': () => + import('@powersync/service-module-mikroorm-storage').then((module) => new module.MikroOrmStorageModule()), postgresql: () => import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule()) } diff --git a/tsconfig.json b/tsconfig.json index e01370b9f..7ad23f53e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -34,6 +34,9 @@ { "path": "./modules/module-postgres-storage" }, + { + "path": "./modules/module-mikroorm-storage" + }, { "path": "./modules/module-mssql" }, @@ -97,6 +100,9 @@ { "path": "./modules/module-postgres-storage/test" }, + { + "path": "./modules/module-mikroorm-storage/test" + }, { "path": "./modules/module-mssql/test" }, From ab1bc2b3a09f9a15bd8714e188c66755fda1006a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 12 Jun 2026 16:09:42 +0200 Subject: [PATCH 02/11] fixes from testing --- modules/module-mikroorm-storage/README.md | 114 ++++++++++++++++++ .../src/drivers/sqlite/sqlite-config.ts | 4 +- .../src/drivers/sqlite/sqlite-dialect.ts | 5 + .../src/storage/MikroOrmBucketBatch.ts | 54 +++++++-- .../storage/MikroOrmBucketStorageFactory.ts | 2 +- .../src/storage/MikroOrmPersistedBatch.ts | 58 +++++++-- .../src/types/types.ts | 8 +- .../test/src/sync-rules-storage.test.ts | 77 ++++++++++++ modules/module-postgres/package.json | 1 + modules/module-postgres/test/src/env.ts | 12 +- modules/module-postgres/test/src/util.ts | 37 ++++++ modules/module-postgres/test/tsconfig.json | 3 + pnpm-lock.yaml | 3 + service/Dockerfile | 6 +- 14 files changed, 355 insertions(+), 29 deletions(-) create mode 100644 modules/module-mikroorm-storage/README.md diff --git a/modules/module-mikroorm-storage/README.md b/modules/module-mikroorm-storage/README.md new file mode 100644 index 000000000..232c660f3 --- /dev/null +++ b/modules/module-mikroorm-storage/README.md @@ -0,0 +1,114 @@ +# MikroORM Bucket Storage + +Experimental MikroORM-backed bucket storage for the PowerSync service. + +This module currently supports SQLite through the `mikroorm:sqlite` storage type. It is intended to prove out a shared MikroORM storage layer that can support more database drivers later, while keeping most bucket storage behavior in common code. + +## Status + +This module is experimental. The SQLite implementation is useful for local development, tests, and validating the MikroORM storage architecture. It is not yet intended as the default production storage backend. + +SQLite storage should only be used when running PowerSync in unified mode. Split API and sync runners are rejected because SQLite checkpoint notifications are process-local and cannot notify separate service processes or pods. + +## How It Works + +The module stores PowerSync bucket state in MikroORM entities: + +- `sync_rules` stores deployed sync rule versions and replication stream state. +- `source_tables` stores source table metadata and snapshot progress. +- `bucket_data` stores bucket operations. +- `bucket_parameters` stores materialized parameter lookups. +- `current_data` stores the latest known source-row state used by write batching and compaction. +- `write_checkpoints` stores custom write checkpoint state. +- `instance` stores the PowerSync storage instance id. + +Common entity definitions live in `src/entities/common`. The SQLite driver reuses those definitions directly instead of redeclaring SQLite-specific entity classes. + +Common storage classes implement most behavior: + +- `MikroOrmBucketStorageFactory` manages sync rule versions and storage instances. +- `MikroOrmSyncRulesStorage` implements the sync-rule-specific storage surface. +- `MikroOrmBucketBatch` owns the writer lifecycle, checkpoints, truncates, and snapshot state. +- `MikroOrmPersistedBatch` persists write chunks in smaller transactions. +- `MikroOrmCompactor` compacts bucket history. +- `MikroOrmStorageDialect` isolates driver-specific streaming and checkpoint notification behavior. + +Migrations are generated and run by MikroORM, but exposed through the standard PowerSync migration surface. Migration execution is guarded by a database-backed lock. For SQLite, that lock table is bootstrapped with raw SQLite before MikroORM migrations run, because migration locking has to work before the normal storage tables exist. + +## Self-Hosted Configuration + +Use `storage.type: mikroorm:sqlite` and provide a SQLite `filename`. + +File-backed SQLite example: + +```yaml +replication: + connections: + - type: postgresql + uri: !env PS_DATA_SOURCE_URI + sslmode: disable + +storage: + type: mikroorm:sqlite + filename: ./powersync-storage.sqlite + +port: 8080 + +sync_rules: + path: sync-rules.yaml + +client_auth: + jwks_uri: !env PS_JWKS_URI + audience: ['powersync'] +``` + +In-memory SQLite example for local tests and throwaway development: + +```yaml +replication: + connections: + - type: postgresql + uri: postgres://postgres:mypassword@localhost:5432/postgres + sslmode: disable + +storage: + type: mikroorm:sqlite + filename: ':memory:' + +port: 8080 + +sync_rules: + path: sync-rules.yaml +``` + +Only use this storage type when starting PowerSync in unified mode. Do not run separate API and sync runners against the same SQLite storage file. + +## Service Registration + +The service image registers this module dynamically under the storage key `mikroorm:sqlite`. The service package depends on `@powersync/service-module-mikroorm-storage`, and `service/src/util/modules.ts` loads `MikroOrmStorageModule` when the config uses this storage type. + +The module also contributes its config type to the generated PowerSync config schema. + +## Development + +Use pnpm through Corepack: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage build +corepack pnpm --filter @powersync/service-module-mikroorm-storage build:tests +corepack pnpm --filter @powersync/service-module-mikroorm-storage test --run +``` + +Focused sync suite: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage test test/src/storage_sync.test.ts --run +``` + +Generate a new SQLite migration from entity changes: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage mikroorm:generate-migration:sqlite +``` + +Review generated migrations before committing them. diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts index 0d71d2381..ec6cae763 100644 --- a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts @@ -12,8 +12,8 @@ export function createSqliteMikroOrmOptions(config: NormalizedMikroOrmSqliteStor entities: sqliteMikroOrmStorageDialect.entityClasses, extensions: [Migrator], migrations: { - path: config.migrations_path ?? SQLITE_MIKRO_ORM_MIGRATIONS_PATH, - pathTs: config.migrations_path ?? SQLITE_MIKRO_ORM_MIGRATIONS_PATH, + path: SQLITE_MIKRO_ORM_MIGRATIONS_PATH, + pathTs: SQLITE_MIKRO_ORM_MIGRATIONS_PATH, glob: '!(*.d).{js,ts}', emit: 'ts', snapshot: false diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts index 394c39de3..9a288c6e7 100644 --- a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts @@ -56,6 +56,11 @@ export const sqliteMikroOrmStorageDialect: MikroOrmStorageDialect = { return; } + // SQLite reads can complete synchronously enough that tight polling loops + // starve replication/checkpoint work in the same process. Yield once before + // the query so single-process unified mode remains cooperative. + await new Promise((resolve) => setImmediate(resolve)); + const filters = options.dataBuckets.map((request) => ({ bucketName: request.bucket, opId: { $gt: request.start, $lte: options.checkpoint } diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts index b39992a6d..eaf35cfd5 100644 --- a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts @@ -77,15 +77,18 @@ export class MikroOrmBucketBatch return em.transactional(async (transactionalEntityManager) => { const existingRows = await transactionalEntityManager.find(this.options.dialect.sourceTableEntity, { groupId: this.options.replicationStreamId, - connectionId: connection_id, - schemaName: schema, - tableName: table + connectionId: connection_id }); let sourceTableRow = existingRows.find((row) => { const matchesRelationId = objectId == null || relationObjectId(row.relationId) == objectId; - return matchesRelationId && jsonEquals(row.replicaIdColumns, normalizedReplicaIdColumns); + return ( + row.schemaName == schema && + row.tableName == table && + matchesRelationId && + jsonEquals(row.replicaIdColumns, normalizedReplicaIdColumns) + ); }) ?? null; if (sourceTableRow == null) { @@ -97,7 +100,7 @@ export class MikroOrmBucketBatch schemaName: schema, tableName: table, replicaIdColumns: normalizedReplicaIdColumns, - snapshotDone: true, + snapshotDone: false, snapshotTotalEstimatedCount: null, snapshotReplicatedCount: null, snapshotLastKey: null @@ -111,7 +114,10 @@ export class MikroOrmBucketBatch const dropTables = existingRows .filter((row) => row.id != sourceTableRow.id) - .filter((row) => objectId == null || relationObjectId(row.relationId) == objectId) + .filter((row) => { + const matchesTableName = row.schemaName == schema && row.tableName == table; + return objectId == null ? matchesTableName : relationObjectId(row.relationId) == objectId || matchesTableName; + }) .map((row) => sourceTableFromRow(row, connectionTag, syncRules)); return { @@ -213,6 +219,12 @@ export class MikroOrmBucketBatch const lastOpId = nextOpId - 1n; this.persistedOp = lastOpId; this.last_flushed_op = lastOpId; + this.logFlushedOperations({ + operationCount: sourceTables.length, + firstOpId, + nextOpId, + operationLabel: 'truncate source table' + }); return { flushed_op: lastOpId }; } @@ -237,6 +249,7 @@ export class MikroOrmBucketBatch const operations = this.pendingOperations.splice(0); const em = this.options.orm.em.fork(); let nextOpId = await this.getNextOpId(em); + const firstOpId = nextOpId; for (const batch of chunked(operations, MAX_OPERATION_BATCH_COUNT)) { await this.options.orm.em.fork().transactional(async (transactionalEntityManager) => { const persistedBatch = this.createPersistedBatch(transactionalEntityManager); @@ -248,6 +261,11 @@ export class MikroOrmBucketBatch this.persistedOp = lastOpId; this.last_flushed_op = lastOpId; result = { flushed_op: lastOpId }; + this.logFlushedOperations({ + operationCount: operations.length, + firstOpId, + nextOpId + }); await this.options.hooks?.afterBatchFlush?.(this); } @@ -507,6 +525,28 @@ export class MikroOrmBucketBatch return maxBigint(bucketData?.opId, bucketParameters?.id, pendingDelete?.pendingDelete, 0n) + 1n; } + private logFlushedOperations(options: { + operationCount: number; + firstOpId: bigint; + nextOpId: bigint; + operationLabel?: string; + }): void { + const storageOperationCount = options.nextOpId - options.firstOpId; + const operationLabel = options.operationLabel ?? 'source operation'; + const pluralizedOperationLabel = options.operationCount == 1 ? operationLabel : `${operationLabel}s`; + + if (storageOperationCount == 0n) { + this.options.logger.info( + `[${this.options.replicationStreamName}] Flushed ${options.operationCount} ${pluralizedOperationLabel} to MikroORM storage DB with no new storage ops` + ); + return; + } + + this.options.logger.info( + `[${this.options.replicationStreamName}] Flushed ${options.operationCount} ${pluralizedOperationLabel} to MikroORM storage DB as ${storageOperationCount.toString()} storage ops (${options.firstOpId.toString()}-${(options.nextOpId - 1n).toString()})` + ); + } + private async markSnapshotDoneInternal(noCheckpointBeforeLsn: string): Promise { const em = this.options.orm.em.fork(); await em.transactional(async (transactionalEntityManager) => { @@ -606,7 +646,7 @@ function normalizeReplicaIdColumns(replicaIdColumns: ColumnDescriptor[]): Column return replicaIdColumns.map((column) => ({ name: column.name, type: column.type, - typeId: column.typeId + typeId: typeof column.typeId === 'undefined' ? column.typeId : Number(column.typeId) })); } diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts index 91c463124..54324cf45 100644 --- a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts @@ -130,7 +130,7 @@ export class MikroOrmBucketStorageFactory extends storage.BucketStorageFactory { } async getReplicatingReplicationStreams(): Promise { - const rows = await this.findSyncRulesRows([storage.SyncRuleState.PROCESSING]); + const rows = await this.findSyncRulesRows([storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]); return rows.map((row) => new MikroOrmPersistedReplicationStream(this.orm, this.dialect, row)); } diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts index 2f3865ad2..7425a9095 100644 --- a/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts @@ -25,6 +25,8 @@ export type CurrentBucket = { id: string; }; +const MAX_ROW_SIZE = 15 * 1024 * 1024; + /** * Handles the writes for a single persisted operation chunk. * @@ -157,7 +159,7 @@ export class MikroOrmPersistedBatch { const storeCurrentData = this.options.storeCurrentData && sourceTable.storeCurrentData; let existingBuckets = currentBuckets(existingCurrentData); let existingLookups = currentLookups(existingCurrentData); - let after = record.after; + let after: sync_rules.ToastableSqliteRow | null | undefined = record.after; if (this.options.skipExistingRows) { if (record.tag == storage.SaveOperationTag.INSERT) { @@ -204,15 +206,24 @@ export class MikroOrmPersistedBatch { let newBuckets: CurrentBucket[] = []; let newLookups: Buffer[] = []; let afterData: Buffer | undefined; + let afterDataWasTruncated = false; if (afterId != null && after != null && utils.isCompleteRow(storeCurrentData, after)) { - afterData = storeCurrentData ? storage.serializeBson(after) : storage.serializeBson({}); + if (storeCurrentData) { + const prepared = this.serializeCurrentData(record, after); + after = prepared.after; + afterData = prepared.data; + afterDataWasTruncated = prepared.truncated; + } else { + afterData = storage.serializeBson({}); + } if (sourceTable.syncData) { - const { results, errors } = this.options.syncRules.evaluateRowWithErrors({ - record: after, + const { results: rawResults, errors } = this.options.syncRules.evaluateRowWithErrors({ + record: after as sync_rules.SqliteRow, sourceTable: sourceTable.ref, bucketDataSources: sourceTable.bucketDataSources }); + const results = afterDataWasTruncated ? rawResults.filter(hasUsableObjectId) : rawResults; for (const error of errors) { this.options.logger.error( `Failed to evaluate data query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` @@ -233,9 +244,13 @@ export class MikroOrmPersistedBatch { } if (sourceTable.syncParameters) { - const { results, errors } = this.options.syncRules.evaluateParameterRowWithErrors(sourceTable.ref, after, { - parameterLookupSources: sourceTable.parameterLookupSources - }); + const { results, errors } = this.options.syncRules.evaluateParameterRowWithErrors( + sourceTable.ref, + after as sync_rules.SqliteRow, + { + parameterLookupSources: sourceTable.parameterLookupSources + } + ); for (const error of errors) { this.options.logger.error( `Failed to evaluate parameter query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` @@ -294,6 +309,31 @@ export class MikroOrmPersistedBatch { } } + private serializeCurrentData( + record: storage.SaveOptions, + after: sync_rules.ToastableSqliteRow + ): { after: sync_rules.ToastableSqliteRow; data: Buffer; truncated: boolean } { + try { + const serialized = storage.serializeBson(after); + if (serialized.byteLength > MAX_ROW_SIZE) { + throw new Error(`Row too large: ${serialized.byteLength}`); + } + return { after, data: serialized, truncated: false }; + } catch (e) { + const error = e instanceof Error ? e : new Error(String(e)); + this.options.logger.warn( + `Data too big on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.message}` + ); + + // Keep the current_data row present, but drop field values. This mirrors + // the Postgres storage behavior for oversized BSON payloads and allows + // future TOAST-style updates to be marked unavailable instead of crashing + // the replication batch. + const emptyValues = Object.fromEntries(Object.keys(after).map((key) => [key, undefined])); + return { after: emptyValues, data: storage.serializeBson(emptyValues), truncated: true }; + } + } + private async upsertCurrentData(options: { tableId: string; sourceKey: storage.ReplicaId; @@ -367,6 +407,10 @@ function currentBucketKey(bucket: CurrentBucket | sync_rules.EvaluatedRow): stri return `${bucket.bucket}/${bucket.table}/${bucket.id}`; } +function hasUsableObjectId(row: sync_rules.EvaluatedRow): boolean { + return row.id !== '' || row.data.id != null; +} + function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string { if (storage.isUUID(id)) { return `${tableId}/${id.toHexString()}`; diff --git a/modules/module-mikroorm-storage/src/types/types.ts b/modules/module-mikroorm-storage/src/types/types.ts index 1ac2271de..5c691a54e 100644 --- a/modules/module-mikroorm-storage/src/types/types.ts +++ b/modules/module-mikroorm-storage/src/types/types.ts @@ -9,11 +9,7 @@ export const MikroOrmSqliteStorageConfig = configFile.BaseStorageConfig.and( /** * SQLite database file. Use ":memory:" for in-memory storage. */ - filename: t.string, - /** - * Optional base directory for MikroORM migrations. - */ - migrations_path: t.string.optional() + filename: t.string }) ); @@ -29,7 +25,6 @@ export const isMikroOrmSqliteStorageConfig = ( export interface NormalizedMikroOrmSqliteStorageConfig { type: typeof MIKRO_ORM_SQLITE_STORAGE_TYPE; filename: string; - migrations_path?: string; max_pool_size: number; } @@ -39,7 +34,6 @@ export function normalizeMikroOrmSqliteStorageConfig( return { type: MIKRO_ORM_SQLITE_STORAGE_TYPE, filename: config.filename, - migrations_path: config.migrations_path, max_pool_size: config.max_pool_size ?? 1 }; } diff --git a/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts b/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts index 45e65dc8f..3ba344c5f 100644 --- a/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts +++ b/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts @@ -96,4 +96,81 @@ bucket_definitions: await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1' })).resolves.toBe(42n); }); + + it('marks newly resolved source tables as requiring an initial snapshot', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +bucket_definitions: + global: + data: + - SELECT * FROM lists +`, + { + validate: false + } + ) + ); + const bucketStorage = factory.getInstance(stream); + await using writer = await bucketStorage.createWriter({ + defaultSchema: 'public', + zeroLSN: '0/0', + storeCurrentData: true + }); + + const resolved = await writer.resolveTables({ + connection_id: 1, + source: { + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: 123, + schema: 'public', + name: 'lists', + replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] + }, + idGenerator: () => 'lists-table' + }); + + expect(resolved.tables[0]?.snapshotComplete).toBe(false); + + await writer.markTableSnapshotDone(resolved.tables, '0/1'); + const resolvedAgain = await writer.resolveTables({ + connection_id: 1, + source: { + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: 123, + schema: 'public', + name: 'lists', + replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] + } + }); + + expect(resolvedAgain.tables[0]?.snapshotComplete).toBe(true); + }); + + it('returns active and processing streams as replicating streams', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId, + state: storage.SyncRuleState.PROCESSING + } + ]); + + const em = factory.orm.em.fork(); + const row = await em.findOneOrFail(factory.dialect.syncRulesEntity, { + id: stream.replicationStreamId + }); + em.assign(row, { state: storage.SyncRuleState.ACTIVE }); + await em.flush(); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId, + state: storage.SyncRuleState.ACTIVE + } + ]); + }); }); diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index 4110c21b8..e6a6d98ce 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -46,6 +46,7 @@ "devDependencies": { "@powersync/lib-service-postgres": "workspace:*", "@powersync/service-core-tests": "workspace:*", + "@powersync/service-module-mikroorm-storage": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-postgres-storage": "workspace:*", "@types/semver": "^7.5.4" diff --git a/modules/module-postgres/test/src/env.ts b/modules/module-postgres/test/src/env.ts index ef3a1e09a..d3b80f2c1 100644 --- a/modules/module-postgres/test/src/env.ts +++ b/modules/module-postgres/test/src/env.ts @@ -1,11 +1,15 @@ import { utils } from '@powersync/lib-services-framework'; export const env = utils.collectEnvironmentVariables({ - PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'), - PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_storage_test'), + PG_TEST_URL: utils.type.string.default('postgres://postgres:mypassword@localhost:5432/powersync_test'), + PG_STORAGE_TEST_URL: utils.type.string.default( + 'postgres://postgres:mypassword@localhost:5432/powersync_storage_test' + ), MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), CI: utils.type.boolean.default('false'), SLOW_TESTS: utils.type.boolean.default('false'), - TEST_MONGO_STORAGE: utils.type.boolean.default('true'), - TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') + TEST_MONGO_STORAGE: utils.type.boolean.default('false'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('false'), + TEST_MIKROORM_SQLITE_STORAGE: utils.type.boolean.default('true'), + MIKROORM_SQLITE_STORAGE_TEST_FILENAME: utils.type.string.default('') }); diff --git a/modules/module-postgres/test/src/util.ts b/modules/module-postgres/test/src/util.ts index 836d770ba..d20195ed2 100644 --- a/modules/module-postgres/test/src/util.ts +++ b/modules/module-postgres/test/src/util.ts @@ -10,8 +10,13 @@ import { TestStorageFactory } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; +import * as mikroorm_storage from '@powersync/service-module-mikroorm-storage'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as postgres_storage from '@powersync/service-module-postgres-storage'; +import { existsSync } from 'node:fs'; +import { unlink } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; import { describe, TestOptions } from 'vitest'; import { env } from './env.js'; @@ -26,6 +31,34 @@ export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils. url: env.PG_STORAGE_TEST_URL }); +const DEFAULT_MIKROORM_SQLITE_STORAGE_FILENAME = join( + tmpdir(), + `powersync-postgres-replication-mikroorm-storage-${process.pid}-${process.env.VITEST_WORKER_ID ?? '0'}.sqlite` +); + +const MIKROORM_SQLITE_STORAGE_FILENAME = + env.MIKROORM_SQLITE_STORAGE_TEST_FILENAME || DEFAULT_MIKROORM_SQLITE_STORAGE_FILENAME; + +export const INITIALIZED_MIKROORM_SQLITE_STORAGE_FACTORY: TestStorageConfig = { + tableIdStrings: true, + factory: async (options) => { + if (!options?.doNotClear && existsSync(MIKROORM_SQLITE_STORAGE_FILENAME)) { + await unlink(MIKROORM_SQLITE_STORAGE_FILENAME); + } + + const config = mikroorm_storage.normalizeMikroOrmSqliteStorageConfig({ + type: mikroorm_storage.MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: MIKROORM_SQLITE_STORAGE_FILENAME + }); + const factory = await mikroorm_storage.createSqliteMikroOrmStorageFactory({ + config, + slotNamePrefix: 'test_' + }); + await factory.orm.schema.update(); + return factory; + } +}; + const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; export interface StorageVersionTestContext { @@ -58,6 +91,10 @@ export function describeWithStorage( if (env.TEST_POSTGRES_STORAGE) { describeFactory('postgres', INITIALIZED_POSTGRES_STORAGE_FACTORY); } + + if (env.TEST_MIKROORM_SQLITE_STORAGE) { + describeFactory('mikroorm sqlite', INITIALIZED_MIKROORM_SQLITE_STORAGE_FACTORY); + } } export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({ diff --git a/modules/module-postgres/test/tsconfig.json b/modules/module-postgres/test/tsconfig.json index b69636818..cf5b32e4c 100644 --- a/modules/module-postgres/test/tsconfig.json +++ b/modules/module-postgres/test/tsconfig.json @@ -19,6 +19,9 @@ { "path": "../../module-mongodb-storage" }, + { + "path": "../../module-mikroorm-storage" + }, { "path": "../../module-postgres-storage" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 368ec7234..f8ed3efa5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -538,6 +538,9 @@ importers: '@powersync/service-core-tests': specifier: workspace:* version: link:../../packages/service-core-tests + '@powersync/service-module-mikroorm-storage': + specifier: workspace:* + version: link:../module-mikroorm-storage '@powersync/service-module-mongodb-storage': specifier: workspace:* version: link:../module-mongodb-storage diff --git a/service/Dockerfile b/service/Dockerfile index 512482297..71726a51f 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -21,6 +21,7 @@ COPY modules/module-postgres/package.json modules/module-postgres/tsconfig.json COPY modules/module-postgres-storage/package.json modules/module-postgres-storage/tsconfig.json modules/module-postgres-storage/ COPY modules/module-mongodb/package.json modules/module-mongodb/tsconfig.json modules/module-mongodb/ COPY modules/module-mongodb-storage/package.json modules/module-mongodb-storage/tsconfig.json modules/module-mongodb-storage/ +COPY modules/module-mikroorm-storage/package.json modules/module-mikroorm-storage/tsconfig.json modules/module-mikroorm-storage/ COPY modules/module-mysql/package.json modules/module-mysql/tsconfig.json modules/module-mysql/ COPY modules/module-mssql/package.json modules/module-mssql/tsconfig.json modules/module-mssql/ COPY modules/module-convex/package.json modules/module-convex/tsconfig.json modules/module-convex/ @@ -50,13 +51,16 @@ COPY modules/module-postgres/sql modules/module-postgres/sql/ COPY modules/module-postgres-storage/src modules/module-postgres-storage/src/ COPY modules/module-mongodb/src modules/module-mongodb/src/ COPY modules/module-mongodb-storage/src modules/module-mongodb-storage/src/ +COPY modules/module-mikroorm-storage/src modules/module-mikroorm-storage/src/ COPY modules/module-mysql/src modules/module-mysql/src/ COPY modules/module-mssql/src modules/module-mssql/src/ COPY modules/module-convex/src modules/module-convex/src/ RUN pnpm build:production && \ rm -rf node_modules **/node_modules && \ - pnpm install --frozen-lockfile --prod --ignore-scripts + pnpm install --frozen-lockfile --prod --ignore-scripts && \ + cd modules/module-mikroorm-storage && \ + pnpm rebuild better-sqlite3 # === PROD === From dc71f7b426c08929e07ded10747553d3c6bb4f0b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 12 Jun 2026 16:29:01 +0200 Subject: [PATCH 03/11] update about concurrency --- modules/module-mikroorm-storage/README.md | 8 + .../src/drivers/sqlite/sqlite-config.ts | 29 ++ .../src/types/types.ts | 2 +- .../test/src/migrations.test.ts | 203 ++++++++++++- plans/mikroorm-bucket-storage.md | 276 ++++++++++++------ 5 files changed, 422 insertions(+), 96 deletions(-) diff --git a/modules/module-mikroorm-storage/README.md b/modules/module-mikroorm-storage/README.md index 232c660f3..54c12e051 100644 --- a/modules/module-mikroorm-storage/README.md +++ b/modules/module-mikroorm-storage/README.md @@ -83,6 +83,14 @@ sync_rules: Only use this storage type when starting PowerSync in unified mode. Do not run separate API and sync runners against the same SQLite storage file. +## SQLite Concurrency + +For file-backed SQLite, the module enables WAL mode and uses MikroORM read replicas to open separate SQLite handles when `storage.max_pool_size` is greater than `1`. This allows SQLite-level read-while-write behavior for API reads during replication writes. + +The current MikroORM SQLite stack uses Kysely's `better-sqlite3` dialect. Those query calls are asynchronous in shape, but they execute on synchronous `better-sqlite3` handles rather than being delegated to worker threads. This means WAL and read replicas improve database handle concurrency, but a long-running SQLite statement can still occupy the Node.js event loop for the process executing it. + +The PowerSync SDK has a Node SQLite dialect that delegates SQLite work to workers. That may be a future workaround if this module needs stronger read concurrency while retaining SQLite storage. + ## Service Registration The service image registers this module dynamically under the storage key `mikroorm:sqlite`. The service package depends on `@powersync/service-module-mikroorm-storage`, and `service/src/util/modules.ts` loads `MikroOrmStorageModule` when the config uses this storage type. diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts index ec6cae763..9a9909968 100644 --- a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts @@ -6,10 +6,20 @@ import { sqliteMikroOrmStorageDialect } from './sqlite-dialect.js'; export const SQLITE_MIKRO_ORM_MIGRATIONS_PATH = new URL('../../migrations/sqlite', import.meta.url).pathname; export function createSqliteMikroOrmOptions(config: NormalizedMikroOrmSqliteStorageConfig) { + const fileBacked = config.filename != ':memory:'; + const readReplicaCount = fileBacked ? Math.max(0, config.max_pool_size - 1) : 0; + return defineConfig({ driver: SqliteDriver, dbName: config.filename, entities: sqliteMikroOrmStorageDialect.entityClasses, + onCreateConnection: (connection) => + configureSqliteConnection(connection, { + enableWal: fileBacked + }), + replicas: Array.from({ length: readReplicaCount }, (_, index) => ({ + name: `reader-${index + 1}` + })), extensions: [Migrator], migrations: { path: SQLITE_MIKRO_ORM_MIGRATIONS_PATH, @@ -24,3 +34,22 @@ export function createSqliteMikroOrmOptions(config: NormalizedMikroOrmSqliteStor export async function createSqliteMikroOrm(config: NormalizedMikroOrmSqliteStorageConfig): Promise { return MikroORM.init(createSqliteMikroOrmOptions(config)); } + +interface SqliteKyselyConnection { + executeQuery(query: { sql: string; parameters: unknown[] }): Promise; +} + +async function configureSqliteConnection( + connection: unknown, + options: { + enableWal: boolean; + } +): Promise { + const sqliteConnection = connection as SqliteKyselyConnection; + + if (options.enableWal) { + await sqliteConnection.executeQuery({ sql: 'PRAGMA journal_mode = WAL', parameters: [] }); + } + + await sqliteConnection.executeQuery({ sql: 'PRAGMA busy_timeout = 5000', parameters: [] }); +} diff --git a/modules/module-mikroorm-storage/src/types/types.ts b/modules/module-mikroorm-storage/src/types/types.ts index 5c691a54e..904cf8c63 100644 --- a/modules/module-mikroorm-storage/src/types/types.ts +++ b/modules/module-mikroorm-storage/src/types/types.ts @@ -34,6 +34,6 @@ export function normalizeMikroOrmSqliteStorageConfig( return { type: MIKRO_ORM_SQLITE_STORAGE_TYPE, filename: config.filename, - max_pool_size: config.max_pool_size ?? 1 + max_pool_size: config.max_pool_size ?? 10 }; } diff --git a/modules/module-mikroorm-storage/test/src/migrations.test.ts b/modules/module-mikroorm-storage/test/src/migrations.test.ts index 1ce98fca3..5f88ab0bc 100644 --- a/modules/module-mikroorm-storage/test/src/migrations.test.ts +++ b/modules/module-mikroorm-storage/test/src/migrations.test.ts @@ -23,7 +23,208 @@ describe('MikroORM migrations', () => { }; afterEach(async () => { - await Promise.all(dbFiles.splice(0).map((file) => rm(file, { force: true }))); + await Promise.all( + dbFiles.splice(0).flatMap((file) => [ + rm(file, { force: true }), + rm(`${file}-shm`, { force: true }), + rm(`${file}-wal`, { force: true }) + ]) + ); + }); + + it('enables WAL and read replicas for file-backed SQLite storage', async () => { + const filename = createDbFile(); + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename, + max_pool_size: 3 + }) + ) + ); + + try { + const driver = orm.em.getDriver(); + const writeConnection = driver.getConnection('write'); + const readConnection = driver.getConnection('read'); + + expect(readConnection).not.toBe(writeConnection); + await expect(writeConnection.execute<{ journal_mode: string }[]>('PRAGMA journal_mode')).resolves.toEqual([ + { journal_mode: 'wal' } + ]); + await expect(readConnection.execute<{ journal_mode: string }[]>('PRAGMA journal_mode')).resolves.toEqual([ + { journal_mode: 'wal' } + ]); + } finally { + await orm.close(true); + } + }); + + it('reads from a replica while the write connection has an open transaction', async () => { + const filename = createDbFile(); + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename, + max_pool_size: 2 + }) + ) + ); + + try { + await orm.em.execute('create table concurrency_test (id integer primary key, value text)', [], 'run'); + await orm.em.execute(`insert into concurrency_test (id, value) values (1, 'committed')`, [], 'run'); + + let releaseWriter!: () => void; + const releaseWriterPromise = new Promise((resolve) => { + releaseWriter = resolve; + }); + let writerReady!: () => void; + const writerReadyPromise = new Promise((resolve) => { + writerReady = resolve; + }); + + const writer = orm.em.fork().transactional(async (transactionalEntityManager) => { + await transactionalEntityManager.execute( + `update concurrency_test set value = 'uncommitted' where id = 1`, + [], + 'run' + ); + writerReady(); + await releaseWriterPromise; + }); + + await writerReadyPromise; + try { + const start = performance.now(); + const rows = await orm.em + .fork() + .getKysely<{ concurrency_test: { id: number; value: string } }>({ type: 'read' }) + .selectFrom('concurrency_test') + .select('value') + .where('id', '=', 1) + .execute(); + const elapsed = performance.now() - start; + + expect(rows).toEqual([{ value: 'committed' }]); + expect(elapsed).toBeLessThan(1_000); + } finally { + releaseWriter(); + await writer; + } + } finally { + await orm.close(true); + } + }); + + it('does not allow overlapping write transactions', async () => { + const filename = createDbFile(); + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename, + max_pool_size: 2 + }) + ) + ); + + try { + await orm.em.execute('create table write_overlap_test (id integer primary key, value text)', [], 'run'); + await orm.em.execute(`insert into write_overlap_test (id, value) values (1, 'first')`, [], 'run'); + + let releaseFirstWriter!: () => void; + const releaseFirstWriterPromise = new Promise((resolve) => { + releaseFirstWriter = resolve; + }); + let firstWriterReady!: () => void; + const firstWriterReadyPromise = new Promise((resolve) => { + firstWriterReady = resolve; + }); + + const firstWriter = orm.em.fork().transactional(async (transactionalEntityManager) => { + await transactionalEntityManager.execute(`update write_overlap_test set value = 'held' where id = 1`, [], 'run'); + firstWriterReady(); + await releaseFirstWriterPromise; + }); + + await firstWriterReadyPromise; + + let secondWriterCompletedBeforeRelease = false; + const secondWriter = orm.em + .fork() + .transactional(async (transactionalEntityManager) => { + await transactionalEntityManager.execute(`update write_overlap_test set value = 'second' where id = 1`, [], 'run'); + }) + .then( + () => ({ status: 'fulfilled' as const }), + (error) => ({ status: 'rejected' as const, error }) + ); + + const secondWriterState = await Promise.race([ + secondWriter.then((result) => { + secondWriterCompletedBeforeRelease = result.status == 'fulfilled'; + return result; + }), + new Promise<{ status: 'pending' }>((resolve) => setTimeout(() => resolve({ status: 'pending' }), 50)) + ]); + + expect(secondWriterCompletedBeforeRelease).toBe(false); + const valueBeforeRelease = await orm.em + .fork() + .getKysely<{ write_overlap_test: { id: number; value: string } }>({ type: 'read' }) + .selectFrom('write_overlap_test') + .select('value') + .where('id', '=', 1) + .execute(); + expect(valueBeforeRelease).toEqual([{ value: 'first' }]); + + releaseFirstWriter(); + await firstWriter; + + if (secondWriterState.status == 'pending') { + await expect(secondWriter).resolves.toEqual({ status: 'fulfilled' }); + } else { + expect(secondWriterState.status).toBe('rejected'); + await orm.em + .fork() + .transactional(async (transactionalEntityManager) => { + await transactionalEntityManager.execute( + `update write_overlap_test set value = 'second' where id = 1`, + [], + 'run' + ); + }); + } + + const finalValue = await orm.em.execute<{ value: string }[]>( + 'select value from write_overlap_test where id = 1' + ); + expect(finalValue).toEqual([{ value: 'second' }]); + } finally { + await orm.close(true); + } + }); + + it('keeps in-memory SQLite storage on a single connection', async () => { + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: ':memory:', + max_pool_size: 3 + }) + ) + ); + + try { + const driver = orm.em.getDriver(); + expect(driver.getConnection('read')).toBe(driver.getConnection('write')); + } finally { + await orm.close(true); + } }); it('runs SQLite migrations through the service migration agent', async () => { diff --git a/plans/mikroorm-bucket-storage.md b/plans/mikroorm-bucket-storage.md index 6537e8660..617dbe81d 100644 --- a/plans/mikroorm-bucket-storage.md +++ b/plans/mikroorm-bucket-storage.md @@ -2,21 +2,28 @@ ## Goal -Build a bucket storage module backed by MikroORM, with SQLite as the first supported driver and a path for future SQL drivers. The module should use common storage logic and common entity definitions wherever possible, while isolating driver-specific concerns behind a small dialect interface. +Build an experimental bucket storage module backed by MikroORM, with SQLite as the first supported driver and a clean path for future SQL drivers. The module must share entity definitions and storage algorithms by default, while isolating database-specific behavior behind small dialect and migration-lock abstractions. -The current first target is `mikroorm:sqlite`. +The first public storage identifier is: -## Lessons Learned +```yaml +storage: + type: mikroorm:sqlite +``` + +SQLite storage is intended for single-process unified PowerSync service deployments only. Future drivers can add cross-process notifications and split-runner support through the dialect layer. + +## Architecture Decisions -- Start from the shared PowerSync storage contract, not from the database schema alone. The quickest validation came from importing the standard core storage and sync tests into the new module. -- MikroORM already handles the primitive types we need well enough for the SQLite first pass. We do not need custom bigint/blob/json property builders for the initial implementation. -- Do not redeclare entities per driver until the driver needs a real difference. Common `defineEntity` schema constants and common classes are enough for SQLite today. -- Driver-specific classes are still useful in a typical MikroORM project when a concrete entity needs methods, hooks, or driver-only behavior. For this module, keep that option open, but do not create empty SQLite subclasses just for ceremony. -- The dialect abstraction should expose concrete entity classes directly, not a nested `MikroOrmStorageEntities` registry. The extra registry layer did not buy much once the common entity classes became stable. -- Advanced read paths should stream. This matters for bucket reads and compaction, where loading all rows into memory is the wrong shape. -- Batch flushes must be split into smaller persisted chunks. A long-lived `BucketBatch` should not keep every record in one MikroORM unit of work. -- SQLite checkpoint notification is process-local only. The provider should reject split service modes for SQLite and require the unified runner, while future drivers can expose cross-process notification. -- Migration locking is a chicken-and-egg problem. The lock must be created before generated MikroORM migrations run, so the lock manager needs a tiny amount of driver-specific raw SQL. +- Start from the PowerSync bucket storage contract, not from a database-first schema design. +- Use MikroORM v7 and its SQL driver stack. +- Use common `defineEntity` schema constants and common TypeScript entity classes for all entities that do not actually vary by driver. +- Do not create empty driver-specific entity subclasses. Add subclasses only when a driver needs methods, hooks, or different behavior. +- Use MikroORM built-in property types for the first SQLite implementation. Avoid custom bigint/blob/json builders unless a later driver proves they are necessary. +- Keep storage algorithms common. Add dialect methods only for query streaming, notification behavior, lock behavior, or raw SQL that cannot be expressed cleanly through typed ORM calls. +- Split high-volume writes into persisted chunks so one `BucketBatch` does not keep every operation in one MikroORM unit of work. +- Stream hot read paths and compaction inputs. Do not load full bucket or compaction result sets into memory. +- Run the shared core storage/sync suites and selected module-postgres replication suites against the MikroORM SQLite storage. ## Module Layout @@ -25,6 +32,7 @@ modules/module-mikroorm-storage/ package.json tsconfig.json vitest.config.ts + README.md src/ index.ts mikro-orm.config.ts @@ -76,18 +84,11 @@ modules/module-mikroorm-storage/ util.ts ``` -## Entity Strategy +## Entity Model -Use common MikroORM `defineEntity` schema constants plus common TypeScript classes: +Use common entity classes with MikroORM `defineEntity` schema constants: ```ts -export class BucketData { - declare id: string; - declare groupId: number; - declare bucketName: string; - declare opId: bigint; -} - export const BucketDataSchema = defineEntity({ name: 'BucketData', tableName: 'bucket_data', @@ -98,10 +99,12 @@ export const BucketDataSchema = defineEntity({ opId: p.bigint('bigint') } }); + +export class BucketData extends BucketDataSchema.class {} BucketDataSchema.setClass(BucketData); ``` -The common entities currently cover: +The common entity set is: - `bucket_data` - `bucket_parameters` @@ -111,11 +114,11 @@ The common entities currently cover: - `sync_rules` - `write_checkpoints` -Keep indexes in the common schema when they are part of the logical access pattern and valid for SQLite. Add driver-specific indexes only when a later driver needs a different shape. +Indexes belong in the common entity schema when they represent logical access patterns shared by drivers. SQLite must include indexes for bucket reads, parameter lookup reads, source-table resolution, current-data lookups, write checkpoints, and sync-rule state queries. -Avoid schema factories like `defineBucketDataEntity(options)` and avoid `property-builders` for now. They made sense when we thought every driver would need custom bigint/blob/json storage, but MikroORM's built-in `p.bigint('bigint')`, `p.blob()`, `p.json()`, `p.string()`, and date handling are enough for the current module. +Source-table metadata must be JSON-safe. Normalize replica id column type ids to `number` before storing them so MikroORM JSON serialization never sees `bigint` values. -Future drivers can still introduce concrete subclasses using the documented MikroORM pattern: +Future driver subclasses should follow the documented MikroORM pattern only when the subclass adds real value: ```ts export class PostgresBucketData extends BucketDataSchema.class { @@ -124,11 +127,9 @@ export class PostgresBucketData extends BucketDataSchema.class { BucketDataSchema.setClass(PostgresBucketData); ``` -Only do that when the subclass has a real purpose. - -## Dialect Strategy +## Dialect Interface -Storage code should depend on `MikroOrmStorageDialect`, not on SQLite imports. The dialect should expose the small set of driver-specific things the common layer needs: +Common storage code depends on `MikroOrmStorageDialect`, not SQLite imports. ```ts export interface MikroOrmStorageDialect { @@ -148,42 +149,53 @@ export interface MikroOrmStorageDialect { The dialect owns: -- the public storage identifier, such as `mikroorm:sqlite` -- the entity classes passed to MikroORM and migration tooling -- streaming query implementation for hot bucket reads +- public storage identifier, for example `mikroorm:sqlite` +- entity class registration for MikroORM and migration tooling +- streamed bucket-data reads - checkpoint watch/notify implementation -- future raw SQL escape hatches where an operation cannot be expressed well through typed ORM calls +- future database-specific raw SQL helpers + +SQLite bucket-data streaming should use MikroORM query builder streaming. It must also yield to the event loop before the query so tight single-process polling loops cannot starve replication/checkpoint work. -Keep common storage classes responsible for the high-level algorithm. Pull database-specific behavior down into dialect methods only when the common implementation would otherwise need driver-specific SQL or notification behavior. +## SQLite Configuration -## SQLite First Pass +SQLite config should be small and explicit: + +```yaml +storage: + type: mikroorm:sqlite + filename: ./powersync-storage.sqlite +``` -SQLite-specific choices: +Rules: -- Use `type: 'mikroorm:sqlite'` as the public storage identifier. -- Use a `db_filename` config value, with `:memory:` supported for fast tests. -- Use MikroORM v7 and its current driver stack. Do not introduce Knex-based helpers. -- Use MikroORM's query builder streaming for bucket reads. -- Use an in-process checkpoint watcher for local wakeups. -- Reject split `api`/`sync` service runners for SQLite storage. SQLite cannot notify sharded pods, so it should only run in unified mode, while command-style contexts used by tooling remain allowed. -- Generate SQLite migrations with the MikroORM CLI instead of hand-writing schema migrations. -- Include indexes needed by the read, write, current-data, source-table, and checkpoint paths in the generated migration. +- `filename` controls the SQLite database file. SQLite creates the file if it does not exist. +- `:memory:` is supported for focused unit tests, but file-backed databases should be used for integration tests that reopen storage with `doNotClear`. +- Do not expose a migrations path in public config. The module owns its bundled migration path. +- Register the module with the service so `mikroorm:sqlite` can be used in normal self-hosted config. +- Include the module in the service Docker build. +- Ensure native `better-sqlite3` bindings are built in the production Docker image. + +SQLite must reject split API/sync runner modes. It can run in unified service mode and command/tooling contexts. ## Migrations -Migrations should use MikroORM migrations internally and be exposed through the standard PowerSync service migration surface. +Use MikroORM migrations internally and expose them through the standard PowerSync migration surface. -Implementation shape: +Implementation requirements: - `MikroOrmMigrationAgent` extends the service migration agent and overrides `run()`. -- The service migration layer triggers the agent, but MikroORM owns migration discovery and migration state. -- `NoOpMigrationStore` exists because PowerSync should not duplicate MikroORM migration bookkeeping. +- The service migration surface triggers the agent. +- MikroORM owns migration discovery and migration state. +- `NoOpMigrationStore` prevents duplicate PowerSync migration bookkeeping. - `AbstractMikroOrmMigrationLockManager` defines the DB-backed lock contract. -- `SqliteMigrationLockManager` bootstraps its own lock table with raw SQLite. +- `SqliteMigrationLockManager` bootstraps its lock table with raw SQLite. + +The raw SQLite lock bootstrap is required. Migration locking has a classic chicken-and-egg problem: migrations normally create tables, but a distributed-safe migration trigger needs a table-backed lock before migrations run. For SQLite, use `CREATE TABLE IF NOT EXISTS` in the lock manager before invoking MikroORM migrations. -The raw SQLite lock bootstrap is intentional. We need a DB-backed lock before migrations run, but migrations are what normally create tables. For SQLite, the lock manager solves that classic chicken-and-egg problem with `CREATE TABLE IF NOT EXISTS` against MikroORM's connection before invoking the migrator. +Generate SQLite migration scripts with the MikroORM CLI. Do not hand-write the initial schema migration except for intentional lock-manager bootstrap SQL. -Tests should verify that the migration agent creates representative storage tables, not just that the migrator returns successfully. +Migration tests must verify that representative storage tables and indexes are created. ## Storage Implementation @@ -198,7 +210,7 @@ Implement the common storage classes in this order: 7. `MikroOrmCompactor` 8. `MikroOrmReportStorage` -`MikroOrmBucketBatch` should own the public writer lifecycle: +`MikroOrmBucketBatch` owns: - listener notifications - `resolveTables` @@ -210,82 +222,158 @@ Implement the common storage classes in this order: - snapshot state - write checkpoints -`MikroOrmPersistedBatch` should own per-transaction write state: +`MikroOrmPersistedBatch` owns: + +- per-transaction `current_data` preload +- bucket data evaluation +- parameter data evaluation +- `bucket_data` inserts +- `bucket_parameters` inserts +- `current_data` upserts and pending-delete markers + +Flush pending operations in bounded chunks, currently `2_000` source operations per persisted transaction. Log successful flushes with source-operation counts and resulting storage-op ranges. + +## Source Tables and Schema Changes + +`resolveTables()` must detect both same-name changes and relation-id changes: + +- Match the active source table by connection, schema, table name, relation id, and normalized replica id columns. +- When a table is renamed, return old source-table rows in `dropTables` if they share the same `relation_id.object_id`. +- When a table changes replica identity or relevant column type metadata, return old source-table rows in `dropTables`. +- New source-table rows must start with `snapshotDone: false` so initial and triggered snapshots are explicit. + +This behavior is required for table recreate, rename, replica identity, and publication-change replication tests. + +## Large Rows and Current Data -- preload `current_data` for a chunk -- evaluate bucket data and parameter rows -- insert `bucket_data` -- insert `bucket_parameters` -- upsert or mark-delete `current_data` +Current-data persistence must tolerate rows that cannot be serialized to BSON or exceed the maximum persisted row size. -Flush pending operations in chunks, currently `2_000` operations per persisted transaction. This keeps high-volume sync tests fast and avoids growing a single MikroORM unit of work across tens of thousands of rows. +Rules: -## Reads, Streaming, and Compaction +- Use a `15 MiB` current-data row limit, matching existing storage behavior. +- If full-row BSON serialization fails or exceeds the limit, log a warning and store a BSON row where each field value is `undefined`. +- Evaluate sync rules against the truncated row so future TOAST-style updates can still be marked unavailable rather than crashing replication. +- Suppress evaluated bucket rows produced only from a truncated row with no usable object id. This prevents placeholder blank-id bucket ops and follow-up blank-id removes. +- Keep legitimate empty-string ids valid when `data.id` is explicitly present. -Bucket reads should stream rows from the dialect through `streamBucketDataRows()`. The common sync path should consume the async iterable and batch output without buffering the entire query result. +## Reads and Checkpoints -Compaction should also stream candidate rows. Use typed ORM calls when they express the operation cleanly. Use raw SQL only for database-specific operations that are awkward or inefficient through MikroORM, and isolate those operations behind common methods or dialect-level helpers. +`MikroOrmSyncRulesStorage.getBucketDataBatch()` should consume dialect-streamed rows and chunk output without buffering the full query result. -The compactor should be validated with the shared sync tests, especially checkpoint invalidation cases. +Bucket read output must include subkeys whenever `source_table` and `source_key` are present. -## Test Strategy +`MikroOrmBucketStorageFactory.getReplicatingReplicationStreams()` must return both processing and active streams. Active streams still replicate after initial snapshot completion and must remain visible to replication management. -The module should import and run the shared core tests instead of building only bespoke SQLite tests. +SQLite checkpoint watching is process-local: -Current expected coverage: +- writes should notify the in-process watcher +- reads should remain cooperative with the event loop +- split-service deployments must be rejected at provider startup +## Compaction + +Compaction should stream candidate rows and avoid loading full result sets into memory. + +Use typed ORM calls where they are readable and efficient. Use raw SQL for database-specific operations that are awkward or inefficient through MikroORM, and isolate that SQL behind common methods or dialect helpers. + +Validate compaction through the shared storage sync tests, especially checkpoint invalidation and bucket batch cases. + +## Tests + +The module must import shared PowerSync storage tests instead of relying only on bespoke SQLite tests. + +Required module-level coverage: + +- migration agent tests +- storage provider tests, including SQLite unified-mode guard - shared storage tests - shared sync tests across storage protocol versions -- migration agent tests - sync-rules storage tests -- provider/service-mode tests +- compaction tests -Known verification commands: +The module-postgres replication tests should be runnable against MikroORM SQLite storage with: + +```sh +TEST_MONGO_STORAGE=false +TEST_POSTGRES_STORAGE=false +TEST_MIKROORM_SQLITE_STORAGE=true +``` + +module-postgres integration tests should use a file-backed SQLite storage filename by default. Allow `MIKROORM_SQLITE_STORAGE_TEST_FILENAME` to override it, but do not default those tests to `:memory:` because resume and `doNotClear` flows need storage state to survive factory reopen. + +Sequential module-postgres files to run against MikroORM SQLite storage: + +- `checkpoints.test.ts` +- `chunked_snapshots.test.ts` +- `large_batch.test.ts` +- `pg_test.test.ts` +- `replica_identity_full.test.ts` +- `replication_retry.test.ts` +- `resuming_snapshots.test.ts` +- `route_api_adapter.test.ts` +- `schema_changes.test.ts` +- `slow_tests.test.ts` +- `storage_combination.test.ts` +- `types/registry.test.ts` +- `validation.test.ts` +- `wal_budget_api.test.ts` +- `wal_budget.test.ts` +- `wal_stream.test.ts` + +Some files are skipped unless their suite-specific environment flags are enabled. + +## Verification Commands ```sh source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage build source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage build:tests -source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage test test/src/storage_sync.test.ts --run source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage test --run +source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-postgres build:tests +source ~/.nvm/nvm.sh && nvm use && TEST_MONGO_STORAGE=false TEST_POSTGRES_STORAGE=false TEST_MIKROORM_SQLITE_STORAGE=true corepack pnpm --filter @powersync/service-module-postgres test test/src/wal_stream.test.ts --run +source ~/.nvm/nvm.sh && nvm use && TEST_MONGO_STORAGE=false TEST_POSTGRES_STORAGE=false TEST_MIKROORM_SQLITE_STORAGE=true corepack pnpm --filter @powersync/service-module-postgres test test/src/schema_changes.test.ts --run ``` -Current observed status after implementing the SQLite vertical slice: - -- source build passes -- test build passes -- sync suite passes with `51 passed` -- full module suite passes with `223 passed | 2 skipped` +Expected baseline: -There has been an intermittent native SQLite/Vitest `139` exit during full-suite startup. Rerunning has passed cleanly, so treat this as runner instability unless a failure includes test-level assertions. +- MikroORM storage source build passes. +- MikroORM storage test build passes. +- Full MikroORM storage suite passes with the shared storage/sync tests. +- module-postgres `wal_stream.test.ts` passes against MikroORM SQLite storage. +- module-postgres `schema_changes.test.ts` passes against MikroORM SQLite storage. ## Build Order From Scratch -1. Create the package, TypeScript config, Vitest config, exports, and workspace wiring. -2. Add the common entity schemas and classes. +1. Create the package, TypeScript config, Vitest config, exports, README, and workspace wiring. +2. Add common MikroORM entity schemas and classes. 3. Add SQLite config and dialect with entity class registration. 4. Configure MikroORM migrations and generate the SQLite initial migration with the MikroORM CLI. 5. Add the DB-backed migration lock and migration agent override. -6. Add provider config for `mikroorm:sqlite` and register it with the service storage engine. -7. Implement factory and sync-rule persistence. -8. Implement writer creation and `MikroOrmBucketBatch`. -9. Split persisted operation writes into `MikroOrmPersistedBatch`. -10. Implement streamed bucket reads through the dialect. -11. Implement write checkpoints and checkpoint watching. -12. Implement compaction with streaming. -13. Add the SQLite unified-runner guard. -14. Import the shared storage and sync tests, then fill gaps until they pass. -15. Run focused sync tests and the full module suite. +6. Add provider config for `mikroorm:sqlite`. +7. Register the module with the service module loader and Docker build. +8. Implement factory and sync-rule persistence. +9. Implement writer creation and `MikroOrmBucketBatch`. +10. Split persisted operation writes into `MikroOrmPersistedBatch`. +11. Implement oversized current-data handling. +12. Implement source-table rename and replica-identity drop detection. +13. Implement streamed bucket reads through the dialect. +14. Implement write checkpoints and checkpoint watching. +15. Implement compaction with streamed reads. +16. Add the SQLite unified-runner guard. +17. Import shared storage and sync tests. +18. Add module-postgres MikroORM SQLite storage test wiring. +19. Run the verification commands and the sequential module-postgres files. ## Future Driver Guidance For Postgres or another SQL driver: -- reuse the common entity definitions first -- add concrete entity subclasses only when methods/hooks/driver behavior require them +- reuse common entity definitions first +- add concrete entity subclasses only for real driver-specific behavior - add a new dialect object with its own entity class list - implement cross-process checkpoint notification in `createCheckpointWatcher()` - move raw SQL into dialect methods when query shape differs by database -- keep migrations generated by MikroORM for that driver -- add driver-specific migration locking before invoking the migrator +- generate migrations with the MikroORM CLI for that driver +- add a DB-backed migration lock before invoking the migrator +- run the shared storage/sync suites and relevant replication-module integration tests against the new driver -This keeps the module abstract without over-abstracting the entity model before a second driver proves what actually varies. +This keeps the module abstract without over-abstracting the entity model before another driver proves what varies. From e71d390d851a8d03839bb1cd96df0e1c93acca21 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 12 Jun 2026 17:21:52 +0200 Subject: [PATCH 04/11] mysql support --- modules/module-mikroorm-storage/README.md | 53 +++++++- modules/module-mikroorm-storage/package.json | 9 +- .../mysql/MySqlMigrationLockManager.ts | 79 +++++++++++ .../mysql/MySqlMikroOrmStorageFactory.ts | 18 +++ .../src/drivers/mysql/mysql-config.ts | 39 ++++++ .../src/drivers/mysql/mysql-dialect.ts | 77 +++++++++++ .../src/drivers/sqlite/sqlite-config.ts | 6 +- .../src/drivers/sqlite/sqlite-dialect.ts | 53 +++----- .../src/entities/common/bucket-data.schema.ts | 23 +--- .../common/bucket-parameters.schema.ts | 15 +-- .../entities/common/current-data.schema.ts | 17 +-- .../src/entities/common/instance.schema.ts | 10 +- .../entities/common/source-table.schema.ts | 21 +-- .../src/entities/common/sync-rules.schema.ts | 38 ++---- .../common/write-checkpoint.schema.ts | 15 +-- .../src/entities/entity-column-types.ts | 43 ++++++ modules/module-mikroorm-storage/src/index.ts | 3 + .../src/migrations/MikroOrmMigrationAgent.ts | 50 +++++-- .../mysql/Migration20260612150058.ts | 29 ++++ ...tion20260612083946_InitialSqliteStorage.ts | 53 -------- .../sqlite/Migration20260612145654.ts | 29 ++++ .../src/mikro-orm.config.ts | 61 +++++++-- .../src/module/MikroOrmStorageModule.ts | 14 +- .../src/storage/MikroOrmStorageDialect.ts | 40 +++++- .../src/storage/MikroOrmStorageProvider.ts | 94 +++++++++---- .../src/types/types.ts | 44 +++++++ .../__snapshots__/mysql-storage.test.ts.snap | 124 ++++++++++++++++++ .../module-mikroorm-storage/test/src/env.ts | 7 + .../test/src/migrations.test.ts | 85 +++++++++++- .../test/src/mysql-storage.test.ts | 46 +++++++ .../test/src/storage-provider.test.ts | 4 +- .../module-mikroorm-storage/test/src/util.ts | 36 +++++ .../schema/src/scripts/compile-json-schema.ts | 4 +- pnpm-lock.yaml | 62 +++++++++ service/src/util/modules.ts | 2 + 35 files changed, 1051 insertions(+), 252 deletions(-) create mode 100644 modules/module-mikroorm-storage/src/drivers/mysql/MySqlMigrationLockManager.ts create mode 100644 modules/module-mikroorm-storage/src/drivers/mysql/MySqlMikroOrmStorageFactory.ts create mode 100644 modules/module-mikroorm-storage/src/drivers/mysql/mysql-config.ts create mode 100644 modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts create mode 100644 modules/module-mikroorm-storage/src/entities/entity-column-types.ts create mode 100644 modules/module-mikroorm-storage/src/migrations/mysql/Migration20260612150058.ts delete mode 100644 modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612083946_InitialSqliteStorage.ts create mode 100644 modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612145654.ts create mode 100644 modules/module-mikroorm-storage/test/src/__snapshots__/mysql-storage.test.ts.snap create mode 100644 modules/module-mikroorm-storage/test/src/env.ts create mode 100644 modules/module-mikroorm-storage/test/src/mysql-storage.test.ts diff --git a/modules/module-mikroorm-storage/README.md b/modules/module-mikroorm-storage/README.md index 54c12e051..e0e886b6e 100644 --- a/modules/module-mikroorm-storage/README.md +++ b/modules/module-mikroorm-storage/README.md @@ -2,7 +2,7 @@ Experimental MikroORM-backed bucket storage for the PowerSync service. -This module currently supports SQLite through the `mikroorm:sqlite` storage type. It is intended to prove out a shared MikroORM storage layer that can support more database drivers later, while keeping most bucket storage behavior in common code. +This module currently supports SQLite through the `mikroorm:sqlite` storage type and has an initial MySQL dialect through `mikroorm:mysql`. It is intended to prove out a shared MikroORM storage layer that can support more database drivers while keeping most bucket storage behavior in common code. ## Status @@ -22,7 +22,7 @@ The module stores PowerSync bucket state in MikroORM entities: - `write_checkpoints` stores custom write checkpoint state. - `instance` stores the PowerSync storage instance id. -Common entity definitions live in `src/entities/common`. The SQLite driver reuses those definitions directly instead of redeclaring SQLite-specific entity classes. +Common entity definitions live in `src/entities/common`. The SQLite and MySQL drivers reuse those definitions directly instead of redeclaring driver-specific entity classes. Common storage classes implement most behavior: @@ -83,6 +83,27 @@ sync_rules: Only use this storage type when starting PowerSync in unified mode. Do not run separate API and sync runners against the same SQLite storage file. +MySQL example: + +```yaml +replication: + connections: + - type: postgresql + uri: !env PS_DATA_SOURCE_URI + sslmode: disable + +storage: + type: mikroorm:mysql + uri: !env PS_STORAGE_MYSQL_URI + +port: 8080 + +sync_rules: + path: sync-rules.yaml +``` + +The MySQL dialect is newer than SQLite and should be treated as experimental. It currently uses the common in-process checkpoint watcher; database-backed notifications can be added behind `MikroOrmStorageDialect` when needed. + ## SQLite Concurrency For file-backed SQLite, the module enables WAL mode and uses MikroORM read replicas to open separate SQLite handles when `storage.max_pool_size` is greater than `1`. This allows SQLite-level read-while-write behavior for API reads during replication writes. @@ -93,7 +114,7 @@ The PowerSync SDK has a Node SQLite dialect that delegates SQLite work to worker ## Service Registration -The service image registers this module dynamically under the storage key `mikroorm:sqlite`. The service package depends on `@powersync/service-module-mikroorm-storage`, and `service/src/util/modules.ts` loads `MikroOrmStorageModule` when the config uses this storage type. +The service image registers this module dynamically under the storage keys `mikroorm:sqlite` and `mikroorm:mysql`. The service package depends on `@powersync/service-module-mikroorm-storage`, and `service/src/util/modules.ts` loads `MikroOrmStorageModule` when the config uses one of these storage types. The module also contributes its config type to the generated PowerSync config schema. @@ -113,10 +134,36 @@ Focused sync suite: corepack pnpm --filter @powersync/service-module-mikroorm-storage test test/src/storage_sync.test.ts --run ``` +Opt into MySQL storage tests by setting a test database URI: + +```sh +MIKROORM_MYSQL_STORAGE_TEST_URI=mysql://repl_user:good_password@localhost:3306/powersync \ + corepack pnpm --filter @powersync/service-module-mikroorm-storage test test/src/mysql-storage.test.ts --run +``` + Generate a new SQLite migration from entity changes: ```sh corepack pnpm --filter @powersync/service-module-mikroorm-storage mikroorm:generate-migration:sqlite ``` +Generate a MySQL migration from entity changes: + +```sh +MIKRO_ORM_MYSQL_URI=mysql://repl_user:good_password@localhost:3306/powersync \ + corepack pnpm --filter @powersync/service-module-mikroorm-storage mikroorm:generate-migration:mysql +``` + +Generate migrations for all supported dialects: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage mikroorm:generate-migrations +``` + +Generate initial migrations for all supported dialects: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage mikroorm:generate-initial-migrations +``` + Review generated migrations before committing them. diff --git a/modules/module-mikroorm-storage/package.json b/modules/module-mikroorm-storage/package.json index 34197788e..657e5a9cc 100644 --- a/modules/module-mikroorm-storage/package.json +++ b/modules/module-mikroorm-storage/package.json @@ -10,10 +10,16 @@ "access": "public" }, "scripts": { - "build": "tsc -b", + "build": "rm -rf ./dist ./tsconfig.tsbuildinfo && tsc -b", "build:tests": "tsc -b test/tsconfig.json", "clean": "rm -rf ./dist && tsc -b --clean", "mikroorm:generate-migration": "pnpm build && mikro-orm migration:create --config dist/mikro-orm.config.js", + "mikroorm:generate-migrations": "pnpm mikroorm:generate-migration:sqlite && pnpm mikroorm:generate-migration:mysql", + "mikroorm:generate-initial-migration": "pnpm build && mikro-orm migration:create --initial --config dist/mikro-orm.config.js", + "mikroorm:generate-initial-migrations": "pnpm mikroorm:generate-initial-migration:sqlite && pnpm mikroorm:generate-initial-migration:mysql", + "mikroorm:generate-initial-migration:mysql": "MIKRO_ORM_STORAGE_DIALECT=mysql pnpm mikroorm:generate-initial-migration", + "mikroorm:generate-initial-migration:sqlite": "MIKRO_ORM_STORAGE_DIALECT=sqlite pnpm mikroorm:generate-initial-migration", + "mikroorm:generate-migration:mysql": "MIKRO_ORM_STORAGE_DIALECT=mysql pnpm mikroorm:generate-migration", "mikroorm:generate-migration:sqlite": "MIKRO_ORM_STORAGE_DIALECT=sqlite pnpm mikroorm:generate-migration", "test": "vitest" }, @@ -34,6 +40,7 @@ "dependencies": { "@mikro-orm/core": "^7.1.4", "@mikro-orm/migrations": "^7.1.4", + "@mikro-orm/mysql": "^7.1.4", "@mikro-orm/sql": "^7.1.4", "@mikro-orm/sqlite": "^7.1.4", "@powersync/lib-services-framework": "workspace:*", diff --git a/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMigrationLockManager.ts b/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMigrationLockManager.ts new file mode 100644 index 000000000..914682e8a --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMigrationLockManager.ts @@ -0,0 +1,79 @@ +import { AbstractMikroOrmMigrationLockManager } from '../../migrations/AbstractMikroOrmMigrationLockManager.js'; + +/** + * MySQL-backed migration lock manager. + * + * The lock table is created with raw SQL because migration locking has to work before MikroORM-created storage tables + * exist. This avoids the classic chicken-and-egg problem where migrations need a lock, but the lock would otherwise + * need a migration-created table. + */ +export class MySqlMigrationLockManager extends AbstractMikroOrmMigrationLockManager { + protected async initLockStore(): Promise { + await this.execute( + ` + CREATE TABLE IF NOT EXISTS powersync_mikroorm_migration_locks ( + name VARCHAR(191) PRIMARY KEY, + lock_id VARCHAR(36) NOT NULL, + expires_at DATETIME(3) NOT NULL, + updated_at DATETIME(3) NOT NULL + ) + `, + [] + ); + } + + protected async tryAcquireLock(options: { + name: string; + lockId: string; + now: Date; + expiresAt: Date; + }): Promise { + const result = await this.execute( + ` + INSERT INTO powersync_mikroorm_migration_locks (name, lock_id, expires_at, updated_at) + VALUES (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + lock_id = IF(expires_at <= ?, VALUES(lock_id), lock_id), + expires_at = IF(expires_at <= ?, VALUES(expires_at), expires_at), + updated_at = IF(expires_at <= ?, VALUES(updated_at), updated_at) + `, + [ + options.name, + options.lockId, + options.expiresAt, + options.now, + options.now, + options.now, + options.now + ] + ); + + return (result.affectedRows ?? 0) > 0; + } + + protected async refreshLock(lockId: string): Promise { + await this.execute( + ` + UPDATE powersync_mikroorm_migration_locks + SET expires_at = ?, updated_at = ? + WHERE name = ? AND lock_id = ? + `, + [new Date(Date.now() + this.timeout), new Date(), this.name, lockId] + ); + } + + protected async releaseLock(lockId: string): Promise { + await this.execute( + ` + DELETE FROM powersync_mikroorm_migration_locks + WHERE name = ? AND lock_id = ? + `, + [this.name, lockId] + ); + } + + private async execute(sql: string, params: unknown[]): Promise<{ affectedRows?: number }> { + const orm = await this.getOrm(); + return orm.em.getConnection().execute(sql, params, 'run'); + } +} diff --git a/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMikroOrmStorageFactory.ts b/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMikroOrmStorageFactory.ts new file mode 100644 index 000000000..707850743 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMikroOrmStorageFactory.ts @@ -0,0 +1,18 @@ +import { MikroORM } from '@mikro-orm/core'; +import { MikroOrmBucketStorageFactory } from '../../storage/MikroOrmBucketStorageFactory.js'; +import { NormalizedMikroOrmMySqlStorageConfig } from '../../types/types.js'; +import { createMySqlMikroOrm } from './mysql-config.js'; +import { mysqlMikroOrmStorageDialect } from './mysql-dialect.js'; + +export async function createMySqlMikroOrmStorageFactory(options: { + config: NormalizedMikroOrmMySqlStorageConfig; + slotNamePrefix: string; + orm?: MikroORM; +}): Promise { + const orm = options.orm ?? (await createMySqlMikroOrm(options.config)); + return new MikroOrmBucketStorageFactory({ + orm, + dialect: mysqlMikroOrmStorageDialect, + slotNamePrefix: options.slotNamePrefix + }); +} diff --git a/modules/module-mikroorm-storage/src/drivers/mysql/mysql-config.ts b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-config.ts new file mode 100644 index 000000000..fc6718e64 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-config.ts @@ -0,0 +1,39 @@ +import { Migrator } from '@mikro-orm/migrations'; +import { defineConfig, MikroORM, MySqlDriver } from '@mikro-orm/mysql'; +import { configureMySqlEntityColumnTypes } from '../../entities/entity-column-types.js'; +import { NormalizedMikroOrmMySqlStorageConfig } from '../../types/types.js'; +import { mysqlMikroOrmStorageDialect } from './mysql-dialect.js'; + +export const MYSQL_MIKRO_ORM_MIGRATIONS_PATH = new URL('../../migrations/mysql', import.meta.url).pathname; + +export function createMySqlMikroOrmOptions(config: NormalizedMikroOrmMySqlStorageConfig) { + configureMySqlEntityColumnTypes(); + + return defineConfig({ + driver: MySqlDriver, + clientUrl: config.uri, + entities: mysqlMikroOrmStorageDialect.entityClasses, + pool: { + min: 0, + max: config.max_pool_size + }, + extensions: [Migrator], + migrations: { + path: MYSQL_MIKRO_ORM_MIGRATIONS_PATH, + pathTs: MYSQL_MIKRO_ORM_MIGRATIONS_PATH, + glob: '!(*.d).{js,ts}', + emit: 'ts', + snapshot: false, + dropTables: false, + // MySQL DDL performs implicit commits, so wrapping generated migrations in + // MikroORM transactions/savepoints can leave the driver trying to roll + // back a savepoint that MySQL has already discarded. + transactional: false, + allOrNothing: false + } + }); +} + +export async function createMySqlMikroOrm(config: NormalizedMikroOrmMySqlStorageConfig): Promise { + return MikroORM.init(createMySqlMikroOrmOptions(config)); +} diff --git a/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts new file mode 100644 index 000000000..0ac570be4 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts @@ -0,0 +1,77 @@ +import type { SqlEntityManager } from '@mikro-orm/sql'; +import { + BucketData, + BucketDataSchema, + BucketParameters, + BucketParametersSchema, + CurrentData, + CurrentDataSchema, + Instance, + InstanceSchema, + SourceTable, + SourceTableSchema, + SyncRules, + SyncRulesSchema, + WriteCheckpoint, + WriteCheckpointSchema +} from '../../entities/entities-index.js'; +import { InProcessMikroOrmCheckpointWatcher, MikroOrmStorageDialect } from '../../storage/MikroOrmStorageDialect.js'; +import { MIKRO_ORM_MYSQL_STORAGE_TYPE } from '../../types/types.js'; + +export const mysqlMikroOrmStorageDialect: MikroOrmStorageDialect = { + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + entityClasses: [ + BucketDataSchema, + BucketParametersSchema, + CurrentDataSchema, + InstanceSchema, + SourceTableSchema, + SyncRulesSchema, + WriteCheckpointSchema + ], + bucketDataEntity: BucketData, + bucketParametersEntity: BucketParameters, + currentDataEntity: CurrentData, + instanceEntity: Instance, + sourceTableEntity: SourceTable, + syncRulesEntity: SyncRules, + writeCheckpointEntity: WriteCheckpoint, + async *streamBucketDataRows(options) { + if (options.dataBuckets.length == 0) { + return; + } + + const filters = options.dataBuckets.map((request) => ({ + bucketName: request.bucket, + opId: { $gt: request.start, $lte: options.checkpoint } + })); + const sqlEntityManager = options.em as SqlEntityManager; + const queryBuilder = sqlEntityManager + .createQueryBuilder(BucketData, 'bucket_data') + .select('*') + .where({ + groupId: options.groupId, + $or: filters + }) + .orderBy({ bucketName: 'ASC', opId: 'ASC' }) + .limit(options.limit); + + yield* streamQueryBuilder(queryBuilder.stream()); + }, + createCheckpointWatcher: () => new InProcessMikroOrmCheckpointWatcher() +}; + +async function* streamQueryBuilder(stream: AsyncIterableIterator): AsyncIterable { + const iterator = stream[Symbol.asyncIterator](); + try { + while (true) { + const result = await iterator.next(); + if (result.done) { + return; + } + yield result.value; + } + } finally { + await iterator.return?.(); + } +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts index 9a9909968..ef70f6c32 100644 --- a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts @@ -1,11 +1,14 @@ import { Migrator } from '@mikro-orm/migrations'; import { defineConfig, MikroORM, SqliteDriver } from '@mikro-orm/sqlite'; +import { configureDefaultEntityColumnTypes } from '../../entities/entity-column-types.js'; import { NormalizedMikroOrmSqliteStorageConfig } from '../../types/types.js'; import { sqliteMikroOrmStorageDialect } from './sqlite-dialect.js'; export const SQLITE_MIKRO_ORM_MIGRATIONS_PATH = new URL('../../migrations/sqlite', import.meta.url).pathname; export function createSqliteMikroOrmOptions(config: NormalizedMikroOrmSqliteStorageConfig) { + configureDefaultEntityColumnTypes(); + const fileBacked = config.filename != ':memory:'; const readReplicaCount = fileBacked ? Math.max(0, config.max_pool_size - 1) : 0; @@ -26,7 +29,8 @@ export function createSqliteMikroOrmOptions(config: NormalizedMikroOrmSqliteStor pathTs: SQLITE_MIKRO_ORM_MIGRATIONS_PATH, glob: '!(*.d).{js,ts}', emit: 'ts', - snapshot: false + snapshot: false, + dropTables: false } }); } diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts index 9a288c6e7..998c0dd3f 100644 --- a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts @@ -1,49 +1,34 @@ import type { SqlEntityManager } from '@mikro-orm/sql'; import { BucketData, + BucketDataSchema, BucketParameters, + BucketParametersSchema, CurrentData, + CurrentDataSchema, Instance, + InstanceSchema, SourceTable, + SourceTableSchema, SyncRules, - WriteCheckpoint + SyncRulesSchema, + WriteCheckpoint, + WriteCheckpointSchema } from '../../entities/entities-index.js'; -import { MikroOrmCheckpointWatcher, MikroOrmStorageDialect } from '../../storage/MikroOrmStorageDialect.js'; +import { InProcessMikroOrmCheckpointWatcher, MikroOrmStorageDialect } from '../../storage/MikroOrmStorageDialect.js'; import { MIKRO_ORM_SQLITE_STORAGE_TYPE } from '../../types/types.js'; -export class SqliteCheckpointWatcher implements MikroOrmCheckpointWatcher { - private readonly listeners = new Set<() => void>(); - - notify(): void { - for (const listener of this.listeners) { - listener(); - } - } - - async *watch(signal: AbortSignal): AsyncIterable { - while (!signal.aborted) { - yield await new Promise((resolve) => { - const listener = () => { - this.listeners.delete(listener); - resolve(); - }; - this.listeners.add(listener); - signal.addEventListener( - 'abort', - () => { - this.listeners.delete(listener); - resolve(); - }, - { once: true } - ); - }); - } - } -} - export const sqliteMikroOrmStorageDialect: MikroOrmStorageDialect = { type: MIKRO_ORM_SQLITE_STORAGE_TYPE, - entityClasses: [BucketData, BucketParameters, CurrentData, Instance, SourceTable, SyncRules, WriteCheckpoint], + entityClasses: [ + BucketDataSchema, + BucketParametersSchema, + CurrentDataSchema, + InstanceSchema, + SourceTableSchema, + SyncRulesSchema, + WriteCheckpointSchema + ], bucketDataEntity: BucketData, bucketParametersEntity: BucketParameters, currentDataEntity: CurrentData, @@ -78,7 +63,7 @@ export const sqliteMikroOrmStorageDialect: MikroOrmStorageDialect = { yield* streamQueryBuilder(queryBuilder.stream()); }, - createCheckpointWatcher: () => new SqliteCheckpointWatcher() + createCheckpointWatcher: () => new InProcessMikroOrmCheckpointWatcher() }; async function* streamQueryBuilder(stream: AsyncIterableIterator): AsyncIterable { diff --git a/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts b/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts index 766018692..bbf7eec71 100644 --- a/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts +++ b/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts @@ -1,19 +1,4 @@ -import { defineEntity, p } from '@mikro-orm/core'; - -export class BucketData { - declare id: string; - declare groupId: number; - declare bucketName: string; - declare opId: bigint; - declare op: string; - declare sourceTable: string | null; - declare sourceKey: Buffer | Uint8Array | null; - declare tableName: string | null; - declare rowId: string | null; - declare checksum: bigint; - declare data: string | null; - declare targetOp: bigint | null; -} +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; export const BucketDataSchema = defineEntity({ name: 'BucketData', @@ -39,8 +24,10 @@ export const BucketDataSchema = defineEntity({ tableName: p.string().fieldName('table_name').nullable(), rowId: p.string().fieldName('row_id').nullable(), checksum: p.bigint('bigint'), - data: p.string().nullable(), + data: p.text().nullable(), targetOp: p.bigint('bigint').nullable() } }); -BucketDataSchema.setClass(BucketData); + +export const BucketData = BucketDataSchema.class; +export type BucketData = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts b/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts index 10f144d13..3a0b6946b 100644 --- a/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts +++ b/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts @@ -1,13 +1,4 @@ -import { defineEntity, p } from '@mikro-orm/core'; - -export class BucketParameters { - declare id: bigint; - declare groupId: number; - declare sourceTable: string; - declare sourceKey: Buffer | Uint8Array; - declare lookup: Buffer | Uint8Array; - declare bucketParameters: unknown; -} +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; export const BucketParametersSchema = defineEntity({ name: 'BucketParameters', @@ -31,4 +22,6 @@ export const BucketParametersSchema = defineEntity({ bucketParameters: p.json() } }); -BucketParametersSchema.setClass(BucketParameters); + +export const BucketParameters = BucketParametersSchema.class; +export type BucketParameters = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts b/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts index 090e14ca3..654888e1f 100644 --- a/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts +++ b/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts @@ -1,15 +1,4 @@ -import { defineEntity, p } from '@mikro-orm/core'; - -export class CurrentData { - declare id: string; - declare groupId: number; - declare sourceTable: string; - declare sourceKey: Buffer | Uint8Array; - declare buckets: unknown; - declare lookups: unknown; - declare data: Buffer | Uint8Array; - declare pendingDelete: bigint | null; -} +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; export const CurrentDataSchema = defineEntity({ name: 'CurrentData', @@ -35,4 +24,6 @@ export const CurrentDataSchema = defineEntity({ pendingDelete: p.bigint('bigint').nullable() } }); -CurrentDataSchema.setClass(CurrentData); + +export const CurrentData = CurrentDataSchema.class; +export type CurrentData = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts b/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts index c38d2d0bc..87c046809 100644 --- a/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts +++ b/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts @@ -1,8 +1,4 @@ -import { defineEntity, p } from '@mikro-orm/core'; - -export class Instance { - declare id: string; -} +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; export const InstanceSchema = defineEntity({ name: 'Instance', @@ -11,4 +7,6 @@ export const InstanceSchema = defineEntity({ id: p.string().primary() } }); -InstanceSchema.setClass(Instance); + +export const Instance = InstanceSchema.class; +export type Instance = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts b/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts index 7c78ab398..4289a9077 100644 --- a/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts +++ b/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts @@ -1,19 +1,4 @@ -import type { Opt } from '@mikro-orm/core'; -import { defineEntity, p } from '@mikro-orm/core'; - -export class SourceTable { - declare id: string; - declare groupId: number; - declare connectionId: number; - declare relationId: unknown | null; - declare schemaName: string; - declare tableName: string; - declare replicaIdColumns: unknown | null; - declare snapshotDone: Opt; - declare snapshotTotalEstimatedCount: bigint | null; - declare snapshotReplicatedCount: bigint | null; - declare snapshotLastKey: Buffer | Uint8Array | null; -} +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; export const SourceTableSchema = defineEntity({ name: 'SourceTable', @@ -38,4 +23,6 @@ export const SourceTableSchema = defineEntity({ snapshotLastKey: p.blob().nullable() } }); -SourceTableSchema.setClass(SourceTable); + +export const SourceTable = SourceTableSchema.class; +export type SourceTable = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts b/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts index de88c7ad8..91f770718 100644 --- a/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts +++ b/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts @@ -1,46 +1,28 @@ -import type { Opt } from '@mikro-orm/core'; -import { defineEntity, p } from '@mikro-orm/core'; +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; import type { storage } from '@powersync/service-core'; -export class SyncRules { - declare id: Opt; - declare state: storage.SyncRuleState; - declare snapshotDone: Opt; - declare snapshotLsn: string | null; - declare lastCheckpoint: bigint | null; - declare lastCheckpointLsn: string | null; - declare noCheckpointBefore: string | null; - declare slotName: string; - declare lastCheckpointTs: Date | null; - declare lastKeepaliveTs: Date | null; - declare lastFatalError: string | null; - declare lastFatalErrorTs: Date | null; - declare keepaliveOp: bigint | null; - declare storageVersion: number | null; - declare content: string; - declare syncPlan: storage.SerializedSyncPlan | null; -} - export const SyncRulesSchema = defineEntity({ name: 'SyncRules', tableName: 'sync_rules', properties: { id: p.integer().primary().autoincrement(), - state: p.string(), + state: p.string().$type(), snapshotDone: p.boolean().fieldName('snapshot_done').default(false), snapshotLsn: p.string().fieldName('snapshot_lsn').nullable(), lastCheckpoint: p.bigint('bigint').nullable(), - lastCheckpointLsn: p.string().fieldName('last_checkpoint_lsn').nullable(), + lastCheckpointLsn: p.string().fieldName('last_checkpoint_lsn').strictNullable(), noCheckpointBefore: p.string().fieldName('no_checkpoint_before').nullable(), slotName: p.string().fieldName('slot_name'), lastCheckpointTs: p.datetime().nullable(), lastKeepaliveTs: p.datetime().nullable(), - lastFatalError: p.string().fieldName('last_fatal_error').nullable(), - lastFatalErrorTs: p.datetime().nullable(), + lastFatalError: p.string().fieldName('last_fatal_error').strictNullable(), + lastFatalErrorTs: p.datetime().strictNullable(), keepaliveOp: p.bigint('bigint').nullable(), storageVersion: p.integer().fieldName('storage_version').nullable(), - content: p.string(), - syncPlan: p.json().nullable() + content: p.text(), + syncPlan: p.json().strictNullable() } }); -SyncRulesSchema.setClass(SyncRules); + +export const SyncRules = SyncRulesSchema.class; +export type SyncRules = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts b/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts index 36b4b5487..a9a304b90 100644 --- a/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts +++ b/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts @@ -1,13 +1,4 @@ -import { defineEntity, p } from '@mikro-orm/core'; - -export class WriteCheckpoint { - declare id: string; - declare syncRulesId: number | null; - declare userId: string; - declare checkpoint: bigint; - declare heads: unknown | null; - declare createdAt: Date; -} +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; export const WriteCheckpointSchema = defineEntity({ name: 'WriteCheckpoint', @@ -27,4 +18,6 @@ export const WriteCheckpointSchema = defineEntity({ createdAt: p.datetime() } }); -WriteCheckpointSchema.setClass(WriteCheckpoint); + +export const WriteCheckpoint = WriteCheckpointSchema.class; +export type WriteCheckpoint = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/entity-column-types.ts b/modules/module-mikroorm-storage/src/entities/entity-column-types.ts new file mode 100644 index 000000000..46bce46c8 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/entity-column-types.ts @@ -0,0 +1,43 @@ +import { + BucketDataSchema, + BucketParametersSchema, + CurrentDataSchema, + SyncRulesSchema +} from './entities-index.js'; + +const MYSQL_INDEXED_BINARY_COLUMN_TYPE = 'varbinary(1024)'; +const MYSQL_LONG_TEXT_COLUMN_TYPE = 'longtext'; +const MYSQL_LONG_BINARY_COLUMN_TYPE = 'longblob'; + +export function configureDefaultEntityColumnTypes(): void { + setColumnType(BucketDataSchema, 'sourceKey', undefined); + setColumnType(BucketDataSchema, 'data', undefined); + setColumnType(BucketParametersSchema, 'sourceKey', undefined); + setColumnType(BucketParametersSchema, 'lookup', undefined); + setColumnType(CurrentDataSchema, 'sourceKey', undefined); + setColumnType(CurrentDataSchema, 'data', undefined); + setColumnType(SyncRulesSchema, 'content', undefined); +} + +export function configureMySqlEntityColumnTypes(): void { + setColumnType(BucketDataSchema, 'sourceKey', MYSQL_INDEXED_BINARY_COLUMN_TYPE); + setColumnType(BucketDataSchema, 'data', MYSQL_LONG_TEXT_COLUMN_TYPE); + setColumnType(BucketParametersSchema, 'sourceKey', MYSQL_INDEXED_BINARY_COLUMN_TYPE); + setColumnType(BucketParametersSchema, 'lookup', MYSQL_INDEXED_BINARY_COLUMN_TYPE); + setColumnType(CurrentDataSchema, 'sourceKey', MYSQL_INDEXED_BINARY_COLUMN_TYPE); + setColumnType(CurrentDataSchema, 'data', MYSQL_LONG_BINARY_COLUMN_TYPE); + setColumnType(SyncRulesSchema, 'content', MYSQL_LONG_TEXT_COLUMN_TYPE); +} + +function setColumnType(schema: { properties: Record }, propertyName: string, columnType: string | undefined) { + const property = schema.properties[propertyName] as { columnType?: string } | undefined; + if (property == null) { + throw new Error(`Cannot configure missing MikroORM entity property ${propertyName}`); + } + + if (columnType == null) { + delete property.columnType; + } else { + property.columnType = columnType; + } +} diff --git a/modules/module-mikroorm-storage/src/index.ts b/modules/module-mikroorm-storage/src/index.ts index 28b08fde4..30dae6129 100644 --- a/modules/module-mikroorm-storage/src/index.ts +++ b/modules/module-mikroorm-storage/src/index.ts @@ -1,3 +1,6 @@ +export * from './drivers/mysql/mysql-config.js'; +export * from './drivers/mysql/mysql-dialect.js'; +export * from './drivers/mysql/MySqlMikroOrmStorageFactory.js'; export * from './drivers/sqlite/sqlite-config.js'; export * from './drivers/sqlite/sqlite-dialect.js'; export * from './drivers/sqlite/SqliteMikroOrmStorageFactory.js'; diff --git a/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts b/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts index b05cf786d..147879c15 100644 --- a/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts +++ b/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts @@ -1,9 +1,17 @@ import { MikroORM } from '@mikro-orm/core'; import * as framework from '@powersync/lib-services-framework'; import { migrations } from '@powersync/service-core'; +import { createMySqlMikroOrm } from '../drivers/mysql/mysql-config.js'; +import { MySqlMigrationLockManager } from '../drivers/mysql/MySqlMigrationLockManager.js'; import { createSqliteMikroOrm } from '../drivers/sqlite/sqlite-config.js'; import { SqliteMigrationLockManager } from '../drivers/sqlite/SqliteMigrationLockManager.js'; -import { MikroOrmSqliteStorageConfigDecoded, normalizeMikroOrmSqliteStorageConfig } from '../types/types.js'; +import { + MIKRO_ORM_MYSQL_STORAGE_TYPE, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + MikroOrmStorageConfigDecoded, + normalizeMikroOrmMySqlStorageConfig, + normalizeMikroOrmSqliteStorageConfig +} from '../types/types.js'; import { NoOpMigrationStore } from './NoOpMigrationStore.js'; /** @@ -18,14 +26,12 @@ export class MikroOrmMigrationAgent extends migrations.AbstractPowerSyncMigratio private readonly ormPromise: Promise; - constructor(config: MikroOrmSqliteStorageConfigDecoded) { + constructor(config: MikroOrmStorageConfigDecoded) { super(); - this.ormPromise = createSqliteMikroOrm(normalizeMikroOrmSqliteStorageConfig(config)); + const runtime = createMigrationRuntime(config); + this.ormPromise = runtime.ormPromise; this.store = new NoOpMigrationStore(); - this.locks = new SqliteMigrationLockManager({ - name: 'mikroorm-migrations', - orm: this.ormPromise - }); + this.locks = runtime.lockManager; } getInternalScriptsDir(): string { @@ -62,9 +68,9 @@ export class MikroOrmMigrationAgent extends migrations.AbstractPowerSyncMigratio } const orm = await this.ormPromise; - const migrator = orm.migrator; logger.info(`Running MikroORM migrations ${params.direction}`); + const migrator = orm.migrator; if (migrator != null) { if (params.direction == framework.Direction.Up) { await migrator.up(); @@ -91,3 +97,31 @@ export class MikroOrmMigrationAgent extends migrations.AbstractPowerSyncMigratio await orm.close(true); } } + +function createMigrationRuntime(config: MikroOrmStorageConfigDecoded): { + ormPromise: Promise; + lockManager: framework.LockManager; +} { + switch (config.type) { + case MIKRO_ORM_SQLITE_STORAGE_TYPE: { + const ormPromise = createSqliteMikroOrm(normalizeMikroOrmSqliteStorageConfig(config)); + return { + ormPromise, + lockManager: new SqliteMigrationLockManager({ + name: 'mikroorm-migrations', + orm: ormPromise + }) + }; + } + case MIKRO_ORM_MYSQL_STORAGE_TYPE: { + const ormPromise = createMySqlMikroOrm(normalizeMikroOrmMySqlStorageConfig(config)); + return { + ormPromise, + lockManager: new MySqlMigrationLockManager({ + name: 'mikroorm-migrations', + orm: ormPromise + }) + }; + } + } +} diff --git a/modules/module-mikroorm-storage/src/migrations/mysql/Migration20260612150058.ts b/modules/module-mikroorm-storage/src/migrations/mysql/Migration20260612150058.ts new file mode 100644 index 000000000..a1ada3de9 --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/mysql/Migration20260612150058.ts @@ -0,0 +1,29 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20260612150058 extends Migration { + + override up(): void | Promise { + this.addSql(`create table \`bucket_data\` (\`id\` varchar(255) not null, \`group_id\` int not null, \`bucket_name\` varchar(255) not null, \`op_id\` bigint not null, \`op\` varchar(255) not null, \`source_table\` varchar(255) null, \`source_key\` varbinary(1024) null, \`table_name\` varchar(255) null, \`row_id\` varchar(255) null, \`checksum\` bigint not null, \`data\` longtext null, \`target_op\` bigint null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`bucket_data\` add index \`bucket_data_bucket_op_index\` (\`group_id\`, \`bucket_name\`, \`op_id\`);`); + this.addSql(`alter table \`bucket_data\` add index \`bucket_data_source_index\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + + this.addSql(`create table \`bucket_parameters\` (\`id\` bigint unsigned not null, \`group_id\` int not null, \`source_table\` varchar(255) not null, \`source_key\` varbinary(1024) not null, \`lookup\` varbinary(1024) not null, \`bucket_parameters\` json not null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`bucket_parameters\` add index \`bucket_parameters_lookup_index\` (\`group_id\`, \`lookup\`, \`id\`);`); + this.addSql(`alter table \`bucket_parameters\` add index \`bucket_parameters_source_index\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + + this.addSql(`create table \`current_data\` (\`id\` varchar(255) not null, \`group_id\` int not null, \`source_table\` varchar(255) not null, \`source_key\` varbinary(1024) not null, \`buckets\` json not null, \`lookups\` json not null, \`data\` longblob not null, \`pending_delete\` bigint null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`current_data\` add index \`current_data_source_index\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + this.addSql(`alter table \`current_data\` add index \`current_data_pending_delete_index\` (\`group_id\`, \`pending_delete\`);`); + + this.addSql(`create table \`instance\` (\`id\` varchar(255) not null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + + this.addSql(`create table \`source_tables\` (\`id\` varchar(255) not null, \`group_id\` int not null, \`connection_id\` int not null, \`relation_id\` json null, \`schema_name\` varchar(255) not null, \`table_name\` varchar(255) not null, \`replica_id_columns\` json null, \`snapshot_done\` tinyint(1) not null default true, \`snapshot_total_estimated_count\` bigint null, \`snapshot_replicated_count\` bigint null, \`snapshot_last_key\` blob null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`source_tables\` add index \`source_table_lookup\` (\`group_id\`, \`table_name\`);`); + + this.addSql(`create table \`sync_rules\` (\`id\` int unsigned not null auto_increment primary key, \`state\` varchar(255) not null, \`snapshot_done\` tinyint(1) not null default false, \`snapshot_lsn\` varchar(255) null, \`last_checkpoint\` bigint null, \`last_checkpoint_lsn\` varchar(255) null, \`no_checkpoint_before\` varchar(255) null, \`slot_name\` varchar(255) not null, \`last_checkpoint_ts\` datetime null, \`last_keepalive_ts\` datetime null, \`last_fatal_error\` varchar(255) null, \`last_fatal_error_ts\` datetime null, \`keepalive_op\` bigint null, \`storage_version\` int null, \`content\` longtext not null, \`sync_plan\` json null) default character set utf8mb4 engine = InnoDB;`); + + this.addSql(`create table \`write_checkpoints\` (\`id\` varchar(255) not null, \`sync_rules_id\` int null, \`user_id\` varchar(255) not null, \`checkpoint\` bigint not null, \`heads\` json null, \`created_at\` datetime not null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`write_checkpoints\` add index \`write_checkpoints_user_checkpoint_index\` (\`user_id\`, \`sync_rules_id\`, \`checkpoint\`);`); + } + +} diff --git a/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612083946_InitialSqliteStorage.ts b/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612083946_InitialSqliteStorage.ts deleted file mode 100644 index 236e92580..000000000 --- a/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612083946_InitialSqliteStorage.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Migration } from '@mikro-orm/migrations'; - -export class Migration20260612083946_InitialSqliteStorage extends Migration { - override async up(): Promise { - this.addSql( - `create table \`bucket_data\` (\`id\` text not null, \`group_id\` integer not null, \`bucket_name\` text not null, \`op_id\` bigint not null, \`op\` text not null, \`source_table\` text null, \`source_key\` blob null, \`table_name\` text null, \`row_id\` text null, \`checksum\` bigint not null, \`data\` text null, \`target_op\` bigint null, primary key (\`id\`));` - ); - this.addSql( - `create index \`bucket_data_bucket_op_index\` on \`bucket_data\` (\`group_id\`, \`bucket_name\`, \`op_id\`);` - ); - this.addSql( - `create index \`bucket_data_source_index\` on \`bucket_data\` (\`group_id\`, \`source_table\`, \`source_key\`);` - ); - - this.addSql( - `create table \`bucket_parameters\` (\`id\` bigint not null, \`group_id\` integer not null, \`source_table\` text not null, \`source_key\` blob not null, \`lookup\` blob not null, \`bucket_parameters\` json not null, primary key (\`id\`));` - ); - this.addSql( - `create index \`bucket_parameters_lookup_index\` on \`bucket_parameters\` (\`group_id\`, \`lookup\`, \`id\`);` - ); - this.addSql( - `create index \`bucket_parameters_source_index\` on \`bucket_parameters\` (\`group_id\`, \`source_table\`, \`source_key\`);` - ); - - this.addSql( - `create table \`current_data\` (\`id\` text not null, \`group_id\` integer not null, \`source_table\` text not null, \`source_key\` blob not null, \`buckets\` json not null, \`lookups\` json not null, \`data\` blob not null, \`pending_delete\` bigint null, primary key (\`id\`));` - ); - this.addSql( - `create index \`current_data_source_index\` on \`current_data\` (\`group_id\`, \`source_table\`, \`source_key\`);` - ); - this.addSql( - `create index \`current_data_pending_delete_index\` on \`current_data\` (\`group_id\`, \`pending_delete\`);` - ); - - this.addSql(`create table \`instance\` (\`id\` text not null, primary key (\`id\`));`); - - this.addSql( - `create table \`source_tables\` (\`id\` text not null, \`group_id\` integer not null, \`connection_id\` integer not null, \`relation_id\` json null, \`schema_name\` text not null, \`table_name\` text not null, \`replica_id_columns\` json null, \`snapshot_done\` integer not null default true, \`snapshot_total_estimated_count\` bigint null, \`snapshot_replicated_count\` bigint null, \`snapshot_last_key\` blob null, primary key (\`id\`));` - ); - this.addSql(`create index \`source_table_lookup\` on \`source_tables\` (\`group_id\`, \`table_name\`);`); - - this.addSql( - `create table \`sync_rules\` (\`id\` integer not null primary key autoincrement, \`state\` text not null, \`snapshot_done\` integer not null default false, \`snapshot_lsn\` text null, \`last_checkpoint\` bigint null, \`last_checkpoint_lsn\` text null, \`no_checkpoint_before\` text null, \`slot_name\` text not null, \`last_checkpoint_ts\` datetime null, \`last_keepalive_ts\` datetime null, \`last_fatal_error\` text null, \`last_fatal_error_ts\` datetime null, \`keepalive_op\` bigint null, \`storage_version\` integer null, \`content\` text not null, \`sync_plan\` json null);` - ); - - this.addSql( - `create table \`write_checkpoints\` (\`id\` text not null, \`sync_rules_id\` integer null, \`user_id\` text not null, \`checkpoint\` bigint not null, \`heads\` json null, \`created_at\` datetime not null, primary key (\`id\`));` - ); - this.addSql( - `create index \`write_checkpoints_user_checkpoint_index\` on \`write_checkpoints\` (\`user_id\`, \`sync_rules_id\`, \`checkpoint\`);` - ); - } -} diff --git a/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612145654.ts b/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612145654.ts new file mode 100644 index 000000000..8f62fe48e --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612145654.ts @@ -0,0 +1,29 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20260612145654 extends Migration { + + override up(): void | Promise { + this.addSql(`create table \`bucket_data\` (\`id\` text not null primary key, \`group_id\` integer not null, \`bucket_name\` text not null, \`op_id\` bigint not null, \`op\` text not null, \`source_table\` text null, \`source_key\` blob null, \`table_name\` text null, \`row_id\` text null, \`checksum\` bigint not null, \`data\` text null, \`target_op\` bigint null);`); + this.addSql(`create index \`bucket_data_bucket_op_index\` on \`bucket_data\` (\`group_id\`, \`bucket_name\`, \`op_id\`);`); + this.addSql(`create index \`bucket_data_source_index\` on \`bucket_data\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + + this.addSql(`create table \`bucket_parameters\` (\`id\` bigint not null primary key, \`group_id\` integer not null, \`source_table\` text not null, \`source_key\` blob not null, \`lookup\` blob not null, \`bucket_parameters\` json not null);`); + this.addSql(`create index \`bucket_parameters_lookup_index\` on \`bucket_parameters\` (\`group_id\`, \`lookup\`, \`id\`);`); + this.addSql(`create index \`bucket_parameters_source_index\` on \`bucket_parameters\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + + this.addSql(`create table \`current_data\` (\`id\` text not null primary key, \`group_id\` integer not null, \`source_table\` text not null, \`source_key\` blob not null, \`buckets\` json not null, \`lookups\` json not null, \`data\` blob not null, \`pending_delete\` bigint null);`); + this.addSql(`create index \`current_data_source_index\` on \`current_data\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + this.addSql(`create index \`current_data_pending_delete_index\` on \`current_data\` (\`group_id\`, \`pending_delete\`);`); + + this.addSql(`create table \`instance\` (\`id\` text not null primary key);`); + + this.addSql(`create table \`source_tables\` (\`id\` text not null primary key, \`group_id\` integer not null, \`connection_id\` integer not null, \`relation_id\` json null, \`schema_name\` text not null, \`table_name\` text not null, \`replica_id_columns\` json null, \`snapshot_done\` integer not null default true, \`snapshot_total_estimated_count\` bigint null, \`snapshot_replicated_count\` bigint null, \`snapshot_last_key\` blob null);`); + this.addSql(`create index \`source_table_lookup\` on \`source_tables\` (\`group_id\`, \`table_name\`);`); + + this.addSql(`create table \`sync_rules\` (\`id\` integer not null primary key autoincrement, \`state\` text not null, \`snapshot_done\` integer not null default false, \`snapshot_lsn\` text null, \`last_checkpoint\` bigint null, \`last_checkpoint_lsn\` text null, \`no_checkpoint_before\` text null, \`slot_name\` text not null, \`last_checkpoint_ts\` datetime null, \`last_keepalive_ts\` datetime null, \`last_fatal_error\` text null, \`last_fatal_error_ts\` datetime null, \`keepalive_op\` bigint null, \`storage_version\` integer null, \`content\` text not null, \`sync_plan\` json null);`); + + this.addSql(`create table \`write_checkpoints\` (\`id\` text not null primary key, \`sync_rules_id\` integer null, \`user_id\` text not null, \`checkpoint\` bigint not null, \`heads\` json null, \`created_at\` datetime not null);`); + this.addSql(`create index \`write_checkpoints_user_checkpoint_index\` on \`write_checkpoints\` (\`user_id\`, \`sync_rules_id\`, \`checkpoint\`);`); + } + +} diff --git a/modules/module-mikroorm-storage/src/mikro-orm.config.ts b/modules/module-mikroorm-storage/src/mikro-orm.config.ts index 86f7261e1..913942429 100644 --- a/modules/module-mikroorm-storage/src/mikro-orm.config.ts +++ b/modules/module-mikroorm-storage/src/mikro-orm.config.ts @@ -1,10 +1,16 @@ -import type { Options } from '@mikro-orm/sqlite'; import { resolve } from 'node:path'; +import { mkdirSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; +import { createMySqlMikroOrmOptions } from './drivers/mysql/mysql-config.js'; import { createSqliteMikroOrmOptions } from './drivers/sqlite/sqlite-config.js'; -import { MIKRO_ORM_SQLITE_STORAGE_TYPE, normalizeMikroOrmSqliteStorageConfig } from './types/types.js'; +import { + MIKRO_ORM_MYSQL_STORAGE_TYPE, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + normalizeMikroOrmMySqlStorageConfig, + normalizeMikroOrmSqliteStorageConfig +} from './types/types.js'; -type MikroOrmStorageDialectName = 'sqlite'; +type MikroOrmStorageDialectName = 'sqlite' | 'mysql'; const moduleRoot = fileURLToPath(new URL('..', import.meta.url)); @@ -13,31 +19,68 @@ function getDialect(): MikroOrmStorageDialectName { if (dialect == 'sqlite') { return dialect; } + if (dialect == 'mysql') { + return dialect; + } throw new Error(`Unsupported MikroORM storage dialect for migration generation: ${dialect}`); } -function sqliteConfig(): Options { +function sqliteConfig() { const options = createSqliteMikroOrmOptions( normalizeMikroOrmSqliteStorageConfig({ type: MIKRO_ORM_SQLITE_STORAGE_TYPE, filename: process.env.MIKRO_ORM_SQLITE_DB ?? ':memory:' }) ); + const path = resolve(moduleRoot, 'dist/migrations/sqlite'); + const pathTs = resolve(moduleRoot, 'src/migrations/sqlite'); + ensureMigrationDirectories(path, pathTs); + + return { + ...options, + migrations: { + ...options.migrations, + path, + pathTs, + emit: 'ts' + } + }; +} + +function mysqlConfig() { + const options = createMySqlMikroOrmOptions( + normalizeMikroOrmMySqlStorageConfig({ + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: process.env.MIKRO_ORM_MYSQL_URI ?? 'mysql://repl_user:good_password@localhost:3306/powersync' + }) + ); + const path = resolve(moduleRoot, 'dist/migrations/mysql'); + const pathTs = resolve(moduleRoot, 'src/migrations/mysql'); + ensureMigrationDirectories(path, pathTs); return { ...options, migrations: { ...options.migrations, - path: resolve(moduleRoot, 'dist/migrations/sqlite'), - pathTs: resolve(moduleRoot, 'src/migrations/sqlite'), + path, + pathTs, emit: 'ts' } }; } +function ensureMigrationDirectories(...paths: string[]): void { + for (const path of paths) { + mkdirSync(path, { recursive: true }); + } +} + const configs = { - sqlite: sqliteConfig -} satisfies Record Options>; + sqlite: sqliteConfig, + mysql: mysqlConfig +} satisfies Record unknown>; + +const selectedConfig: unknown = configs[getDialect()](); -export default configs[getDialect()](); +export default selectedConfig; diff --git a/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts b/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts index f9f131e97..edd5908eb 100644 --- a/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts +++ b/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts @@ -1,7 +1,12 @@ import { modules, system } from '@powersync/service-core'; import { MikroOrmMigrationAgent } from '../migrations/MikroOrmMigrationAgent.js'; import { MikroOrmStorageProvider } from '../storage/MikroOrmStorageProvider.js'; -import { isMikroOrmSqliteStorageConfig, MikroOrmSqliteStorageConfig } from '../types/types.js'; +import { + isMikroOrmStorageConfig, + MIKRO_ORM_MYSQL_STORAGE_TYPE, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + MikroOrmStorageConfig +} from '../types/types.js'; export class MikroOrmStorageModule extends modules.AbstractModule { constructor() { @@ -11,11 +16,12 @@ export class MikroOrmStorageModule extends modules.AbstractModule { } async initialize(context: system.ServiceContextContainer): Promise { - context.storageEngine.registerProvider(new MikroOrmStorageProvider()); + context.storageEngine.registerProvider(new MikroOrmStorageProvider(MIKRO_ORM_SQLITE_STORAGE_TYPE)); + context.storageEngine.registerProvider(new MikroOrmStorageProvider(MIKRO_ORM_MYSQL_STORAGE_TYPE)); - if (isMikroOrmSqliteStorageConfig(context.configuration.storage)) { + if (isMikroOrmStorageConfig(context.configuration.storage)) { context.migrations.registerMigrationAgent( - new MikroOrmMigrationAgent(MikroOrmSqliteStorageConfig.decode(context.configuration.storage)) + new MikroOrmMigrationAgent(MikroOrmStorageConfig.decode(context.configuration.storage)) ); } } diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts index 0678cce03..c51a8d658 100644 --- a/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts @@ -1,4 +1,4 @@ -import type { EntityClass, EntityManager } from '@mikro-orm/core'; +import type { EntityClass, EntityManager, EntityName } from '@mikro-orm/core'; import type { BucketDataRequest } from '@powersync/service-core'; import type { BucketData, @@ -17,7 +17,7 @@ export interface MikroOrmStorageDialect { /** Public storage type identifier, e.g. `mikroorm:sqlite`. */ readonly type: string; /** Flat entity class list passed to MikroORM initialization and migration tooling. */ - readonly entityClasses: EntityClass[]; + readonly entityClasses: EntityName[]; /** Stored bucket operations, typically read by bucket and operation id. */ readonly bucketDataEntity: EntityClass; /** Materialized parameter lookups for bucket parameter queries. */ @@ -63,3 +63,39 @@ export interface MikroOrmCheckpointWatcher { /** Yield whenever a checkpoint change should be re-read. */ watch(signal: AbortSignal): AsyncIterable; } + +/** + * Process-local checkpoint watcher for drivers without a database notification implementation. + * + * This is sufficient for single-process runners and tests. Drivers that support cross-process notifications should + * replace this with a watcher backed by the database or an external event channel. + */ +export class InProcessMikroOrmCheckpointWatcher implements MikroOrmCheckpointWatcher { + private readonly listeners = new Set<() => void>(); + + notify(): void { + for (const listener of this.listeners) { + listener(); + } + } + + async *watch(signal: AbortSignal): AsyncIterable { + while (!signal.aborted) { + yield await new Promise((resolve) => { + const listener = () => { + this.listeners.delete(listener); + resolve(); + }; + this.listeners.add(listener); + signal.addEventListener( + 'abort', + () => { + this.listeners.delete(listener); + resolve(); + }, + { once: true } + ); + }); + } + } +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts index 4ccc63143..fecc29f2d 100644 --- a/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts @@ -1,56 +1,96 @@ import { logger } from '@powersync/lib-services-framework'; import { storage, system } from '@powersync/service-core'; +import { createMySqlMikroOrm } from '../drivers/mysql/mysql-config.js'; +import { createMySqlMikroOrmStorageFactory } from '../drivers/mysql/MySqlMikroOrmStorageFactory.js'; import { createSqliteMikroOrm } from '../drivers/sqlite/sqlite-config.js'; import { createSqliteMikroOrmStorageFactory } from '../drivers/sqlite/SqliteMikroOrmStorageFactory.js'; import { + isMikroOrmMySqlStorageConfig, isMikroOrmSqliteStorageConfig, + MIKRO_ORM_MYSQL_STORAGE_TYPE, MIKRO_ORM_SQLITE_STORAGE_TYPE, + MikroOrmMySqlStorageConfig, MikroOrmSqliteStorageConfig, + normalizeMikroOrmMySqlStorageConfig, normalizeMikroOrmSqliteStorageConfig } from '../types/types.js'; import { MikroOrmReportStorage } from './MikroOrmReportStorage.js'; export class MikroOrmStorageProvider implements storage.StorageProvider { + constructor(private readonly storageType: typeof MIKRO_ORM_SQLITE_STORAGE_TYPE | typeof MIKRO_ORM_MYSQL_STORAGE_TYPE) {} + get type() { - return MIKRO_ORM_SQLITE_STORAGE_TYPE; + return this.storageType; } async getStorage(options: storage.GetStorageOptions): Promise { const { storage: storageConfig } = options.resolvedConfig; - if (!isMikroOrmSqliteStorageConfig(storageConfig)) { - throw new Error(`Cannot create MikroORM SQLite storage with provided config ${storageConfig.type}`); + if (this.storageType == MIKRO_ORM_SQLITE_STORAGE_TYPE && isMikroOrmSqliteStorageConfig(storageConfig)) { + assertSqliteServiceMode(options.serviceMode); + + const decodedConfig = MikroOrmSqliteStorageConfig.decode(storageConfig); + const normalizedConfig = normalizeMikroOrmSqliteStorageConfig(decodedConfig); + const orm = await createSqliteMikroOrm(normalizedConfig); + await orm.schema.update(); + + const storageFactory = await createSqliteMikroOrmStorageFactory({ + config: normalizedConfig, + slotNamePrefix: options.resolvedConfig.slot_name_prefix, + orm + }); + + return activeStorage({ + storageFactory, + tearDownLabel: `MikroORM SQLite storage: ${normalizedConfig.filename}`, + tearDown: () => orm.schema.drop() + }); } - assertSqliteServiceMode(options.serviceMode); - const decodedConfig = MikroOrmSqliteStorageConfig.decode(storageConfig); - const normalizedConfig = normalizeMikroOrmSqliteStorageConfig(decodedConfig); - const orm = await createSqliteMikroOrm(normalizedConfig); - await orm.schema.update(); + if (this.storageType == MIKRO_ORM_MYSQL_STORAGE_TYPE && isMikroOrmMySqlStorageConfig(storageConfig)) { + const decodedConfig = MikroOrmMySqlStorageConfig.decode(storageConfig); + const normalizedConfig = normalizeMikroOrmMySqlStorageConfig(decodedConfig); + const orm = await createMySqlMikroOrm(normalizedConfig); + await orm.schema.update(); - const storageFactory = await createSqliteMikroOrmStorageFactory({ - config: normalizedConfig, - slotNamePrefix: options.resolvedConfig.slot_name_prefix, - orm - }); - const reportStorage = new MikroOrmReportStorage(); + const storageFactory = await createMySqlMikroOrmStorageFactory({ + config: normalizedConfig, + slotNamePrefix: options.resolvedConfig.slot_name_prefix, + orm + }); - return { - reportStorage, - storage: storageFactory, - shutDown: async () => { - await storageFactory[Symbol.asyncDispose](); - }, - tearDown: async () => { - logger.info(`Tearing down MikroORM SQLite storage: ${normalizedConfig.filename}...`); - await orm.schema.drop(); - await storageFactory[Symbol.asyncDispose](); - return true; - } - }; + return activeStorage({ + storageFactory, + tearDownLabel: 'MikroORM MySQL storage', + tearDown: () => orm.schema.drop() + }); + } + + throw new Error(`Cannot create ${this.storageType} storage with provided config ${storageConfig.type}`); } } +function activeStorage(options: { + storageFactory: storage.BucketStorageFactory; + tearDownLabel: string; + tearDown: () => Promise; +}): storage.ActiveStorage { + const reportStorage = new MikroOrmReportStorage(); + return { + reportStorage, + storage: options.storageFactory, + shutDown: async () => { + await options.storageFactory[Symbol.asyncDispose](); + }, + tearDown: async () => { + logger.info(`Tearing down ${options.tearDownLabel}...`); + await options.tearDown(); + await options.storageFactory[Symbol.asyncDispose](); + return true; + } + }; +} + const SQLITE_ALLOWED_COMMAND_MODES = new Set([ system.ServiceContextMode.COMPACT, system.ServiceContextMode.TEARDOWN, diff --git a/modules/module-mikroorm-storage/src/types/types.ts b/modules/module-mikroorm-storage/src/types/types.ts index 904cf8c63..ca087ac5b 100644 --- a/modules/module-mikroorm-storage/src/types/types.ts +++ b/modules/module-mikroorm-storage/src/types/types.ts @@ -2,6 +2,7 @@ import { configFile } from '@powersync/service-types'; import * as t from 'ts-codec'; export const MIKRO_ORM_SQLITE_STORAGE_TYPE = 'mikroorm:sqlite'; +export const MIKRO_ORM_MYSQL_STORAGE_TYPE = 'mikroorm:mysql'; export const MikroOrmSqliteStorageConfig = configFile.BaseStorageConfig.and( t.object({ @@ -16,18 +17,51 @@ export const MikroOrmSqliteStorageConfig = configFile.BaseStorageConfig.and( export type MikroOrmSqliteStorageConfig = t.Encoded; export type MikroOrmSqliteStorageConfigDecoded = t.Decoded; +export const MikroOrmMySqlStorageConfig = configFile.BaseStorageConfig.and( + t.object({ + type: t.literal(MIKRO_ORM_MYSQL_STORAGE_TYPE), + /** + * MySQL connection URI, for example mysql://root:password@localhost:3306/powersync_storage. + */ + uri: t.string + }) +); + +export type MikroOrmMySqlStorageConfig = t.Encoded; +export type MikroOrmMySqlStorageConfigDecoded = t.Decoded; + +export const MikroOrmStorageConfig = MikroOrmSqliteStorageConfig.or(MikroOrmMySqlStorageConfig); +export type MikroOrmStorageConfig = t.Encoded; +export type MikroOrmStorageConfigDecoded = t.Decoded; + export const isMikroOrmSqliteStorageConfig = ( config: configFile.GenericStorageConfig ): config is MikroOrmSqliteStorageConfig => { return config.type == MIKRO_ORM_SQLITE_STORAGE_TYPE; }; +export const isMikroOrmMySqlStorageConfig = ( + config: configFile.GenericStorageConfig +): config is MikroOrmMySqlStorageConfig => { + return config.type == MIKRO_ORM_MYSQL_STORAGE_TYPE; +}; + +export const isMikroOrmStorageConfig = (config: configFile.GenericStorageConfig): config is MikroOrmStorageConfig => { + return isMikroOrmSqliteStorageConfig(config) || isMikroOrmMySqlStorageConfig(config); +}; + export interface NormalizedMikroOrmSqliteStorageConfig { type: typeof MIKRO_ORM_SQLITE_STORAGE_TYPE; filename: string; max_pool_size: number; } +export interface NormalizedMikroOrmMySqlStorageConfig { + type: typeof MIKRO_ORM_MYSQL_STORAGE_TYPE; + uri: string; + max_pool_size: number; +} + export function normalizeMikroOrmSqliteStorageConfig( config: MikroOrmSqliteStorageConfigDecoded ): NormalizedMikroOrmSqliteStorageConfig { @@ -37,3 +71,13 @@ export function normalizeMikroOrmSqliteStorageConfig( max_pool_size: config.max_pool_size ?? 10 }; } + +export function normalizeMikroOrmMySqlStorageConfig( + config: MikroOrmMySqlStorageConfigDecoded +): NormalizedMikroOrmMySqlStorageConfig { + return { + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: config.uri, + max_pool_size: config.max_pool_size ?? 10 + }; +} diff --git a/modules/module-mikroorm-storage/test/src/__snapshots__/mysql-storage.test.ts.snap b/modules/module-mikroorm-storage/test/src/__snapshots__/mysql-storage.test.ts.snap new file mode 100644 index 000000000..cdaac777d --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/__snapshots__/mysql-storage.test.ts.snap @@ -0,0 +1,124 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v1 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v1 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v2 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v2 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v3 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v3 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Data - v1 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Data - v2 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Data - v3 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; diff --git a/modules/module-mikroorm-storage/test/src/env.ts b/modules/module-mikroorm-storage/test/src/env.ts new file mode 100644 index 000000000..ac22dc7f0 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/env.ts @@ -0,0 +1,7 @@ +import { utils } from '@powersync/lib-services-framework'; + +export const env = utils.collectEnvironmentVariables({ + MIKROORM_MYSQL_STORAGE_TEST_URI: utils.type.string.default( + process.env.MYSQL_TEST_URI ?? 'mysql://repl_user:good_password@localhost:3306/powersync' + ) +}); diff --git a/modules/module-mikroorm-storage/test/src/migrations.test.ts b/modules/module-mikroorm-storage/test/src/migrations.test.ts index 5f88ab0bc..69b422612 100644 --- a/modules/module-mikroorm-storage/test/src/migrations.test.ts +++ b/modules/module-mikroorm-storage/test/src/migrations.test.ts @@ -3,15 +3,18 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { MikroORM } from '@mikro-orm/core'; +import { MySqlDriver } from '@mikro-orm/mysql'; import { SqliteDriver } from '@mikro-orm/sqlite'; import { Direction } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import { afterEach, describe, expect, it } from 'vitest'; +import { createMySqlMikroOrmOptions } from '../../src/drivers/mysql/mysql-config.js'; import { createSqliteMikroOrmOptions } from '../../src/drivers/sqlite/sqlite-config.js'; import { sqliteMikroOrmStorageDialect } from '../../src/drivers/sqlite/sqlite-dialect.js'; -import { MIKRO_ORM_SQLITE_STORAGE_TYPE } from '../../src/index.js'; +import { MIKRO_ORM_MYSQL_STORAGE_TYPE, MIKRO_ORM_SQLITE_STORAGE_TYPE } from '../../src/index.js'; import { MikroOrmMigrationAgent } from '../../src/migrations/MikroOrmMigrationAgent.js'; -import { normalizeMikroOrmSqliteStorageConfig } from '../../src/types/types.js'; +import { normalizeMikroOrmMySqlStorageConfig, normalizeMikroOrmSqliteStorageConfig } from '../../src/types/types.js'; +import { env } from './env.js'; describe('MikroORM migrations', () => { const dbFiles: string[] = []; @@ -278,6 +281,66 @@ describe('MikroORM migrations', () => { } }); + it('configures MySQL migrations outside MikroORM transactions', () => { + const options = createMySqlMikroOrmOptions( + normalizeMikroOrmMySqlStorageConfig({ + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: env.MIKROORM_MYSQL_STORAGE_TEST_URI || 'mysql://repl_user:good_password@localhost:3306/powersync' + }) + ); + + expect(options.migrations?.transactional).toBe(false); + expect(options.migrations?.allOrNothing).toBe(false); + }); + + it.skipIf(!env.MIKROORM_MYSQL_STORAGE_TEST_URI)( + 'runs MySQL migrations through the service migration agent without transactional DDL', + async () => { + const config = { + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: env.MIKROORM_MYSQL_STORAGE_TEST_URI + } as const; + const normalizedConfig = normalizeMikroOrmMySqlStorageConfig(config); + const setupOrm = await MikroORM.init(createMySqlMikroOrmOptions(normalizedConfig)); + + try { + await dropMySqlMigrationTables(setupOrm); + } finally { + await setupOrm.close(true); + } + + await using agent = new MikroOrmMigrationAgent(config); + + await agent.run({ + direction: Direction.Up, + migrations: [] + }); + + const orm = await MikroORM.init(createMySqlMikroOrmOptions(normalizedConfig)); + + try { + const tables = await orm.em.getConnection().execute<{ tableName: string }[]>( + ` + select table_name as tableName + from information_schema.tables + where table_schema = database() + order by table_name + ` + ); + const tableNames = tables.map((row) => row.tableName); + + expect(tableNames).toContain('powersync_mikroorm_migration_locks'); + expect(tableNames).toContain('mikro_orm_migrations'); + expect(tableNames).toContain('instance'); + expect(tableNames).toContain('sync_rules'); + expect(tableNames).toContain('bucket_data'); + expect(tableNames).toContain('write_checkpoints'); + } finally { + await orm.close(true); + } + } + ); + it('hydrates SQLite-specific storage columns as service-facing values', async () => { const filename = createDbFile(); @@ -373,3 +436,21 @@ describe('MikroORM migrations', () => { } }); }); + +async function dropMySqlMigrationTables(orm: MikroORM): Promise { + const connection = orm.em.getConnection(); + + for (const table of [ + 'write_checkpoints', + 'bucket_parameters', + 'current_data', + 'bucket_data', + 'source_tables', + 'sync_rules', + 'instance', + 'mikro_orm_migrations', + 'powersync_mikroorm_migration_locks' + ]) { + await connection.execute(`drop table if exists \`${table}\``, [], 'run'); + } +} diff --git a/modules/module-mikroorm-storage/test/src/mysql-storage.test.ts b/modules/module-mikroorm-storage/test/src/mysql-storage.test.ts new file mode 100644 index 000000000..5d8a59313 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/mysql-storage.test.ts @@ -0,0 +1,46 @@ +import { register } from '@powersync/service-core-tests'; +import { updateSyncRulesFromYaml } from '@powersync/service-core'; +import { describe, expect, it } from 'vitest'; +import type { MikroOrmBucketStorageFactory } from '../../src/index.js'; +import { env } from './env.js'; +import { MIKRO_ORM_MYSQL_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe.skipIf(!env.MIKROORM_MYSQL_STORAGE_TEST_URI).sequential('MikroORM MySQL Sync Bucket Storage', () => { + it('creates the schema and stores sync rules', async () => { + await using factory = (await MIKRO_ORM_MYSQL_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +bucket_definitions: + global: + data: [] +`, + { + validate: false + } + ) + ); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId + } + ]); + }); + + for (let storageVersion of TEST_STORAGE_VERSIONS) { + describe(`Parameters - v${storageVersion}`, () => + register.registerDataStorageParameterTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion })); + + describe(`Data - v${storageVersion}`, () => + register.registerDataStorageDataTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion })); + + describe(`Checkpoints - v${storageVersion}`, () => + register.registerDataStorageCheckpointTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion })); + + describe(`Compaction - v${storageVersion}`, () => { + register.registerCompactTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion }); + register.registerParameterCompactTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion }); + }); + } +}); diff --git a/modules/module-mikroorm-storage/test/src/storage-provider.test.ts b/modules/module-mikroorm-storage/test/src/storage-provider.test.ts index db21c4c17..8d5453ab2 100644 --- a/modules/module-mikroorm-storage/test/src/storage-provider.test.ts +++ b/modules/module-mikroorm-storage/test/src/storage-provider.test.ts @@ -19,7 +19,7 @@ function getStorageOptions(serviceMode: system.ServiceContextMode): storage.GetS describe('MikroORM SQLite storage provider', () => { it('rejects split service runners', async () => { - const provider = new MikroOrmStorageProvider(); + const provider = new MikroOrmStorageProvider(MIKRO_ORM_SQLITE_STORAGE_TYPE); await expect(provider.getStorage(getStorageOptions(system.ServiceContextMode.API))).rejects.toThrow( 'MikroORM SQLite storage only supports the unified service runner' @@ -30,7 +30,7 @@ describe('MikroORM SQLite storage provider', () => { }); it('allows the unified service runner', async () => { - const provider = new MikroOrmStorageProvider(); + const provider = new MikroOrmStorageProvider(MIKRO_ORM_SQLITE_STORAGE_TYPE); const activeStorage = await provider.getStorage(getStorageOptions(system.ServiceContextMode.UNIFIED)); await activeStorage.shutDown(); diff --git a/modules/module-mikroorm-storage/test/src/util.ts b/modules/module-mikroorm-storage/test/src/util.ts index f04bb025d..105e6c6b8 100644 --- a/modules/module-mikroorm-storage/test/src/util.ts +++ b/modules/module-mikroorm-storage/test/src/util.ts @@ -1,9 +1,13 @@ import { storage, SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core'; import { + createMySqlMikroOrmStorageFactory, createSqliteMikroOrmStorageFactory, + MIKRO_ORM_MYSQL_STORAGE_TYPE, MIKRO_ORM_SQLITE_STORAGE_TYPE, + normalizeMikroOrmMySqlStorageConfig, normalizeMikroOrmSqliteStorageConfig } from '../../src/index.js'; +import { env } from './env.js'; const BASE_CONFIG = { type: MIKRO_ORM_SQLITE_STORAGE_TYPE, @@ -23,4 +27,36 @@ export const MIKRO_ORM_SQLITE_STORAGE_FACTORY: storage.TestStorageConfig = { } }; +export const MIKRO_ORM_MYSQL_STORAGE_FACTORY: storage.TestStorageConfig = { + tableIdStrings: true, + factory: async () => { + const config = normalizeMikroOrmMySqlStorageConfig({ + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: env.MIKROORM_MYSQL_STORAGE_TEST_URI + }); + const factory = await createMySqlMikroOrmStorageFactory({ + config, + slotNamePrefix: 'test_' + }); + await dropMySqlStorageTables(factory.orm); + await factory.orm.schema.create(); + return factory; + } +}; + export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; + +async function dropMySqlStorageTables(orm: Awaited>['orm']) { + const connection = orm.em.getConnection(); + for (const table of [ + 'write_checkpoints', + 'bucket_parameters', + 'current_data', + 'bucket_data', + 'source_tables', + 'sync_rules', + 'instance' + ]) { + await connection.execute(`drop table if exists \`${table}\``, [], 'run'); + } +} diff --git a/packages/schema/src/scripts/compile-json-schema.ts b/packages/schema/src/scripts/compile-json-schema.ts index e5702c981..5f584a049 100644 --- a/packages/schema/src/scripts/compile-json-schema.ts +++ b/packages/schema/src/scripts/compile-json-schema.ts @@ -1,5 +1,5 @@ import { ConvexConnectionConfig } from '@powersync/service-module-convex/types'; -import { MikroOrmSqliteStorageConfig } from '@powersync/service-module-mikroorm-storage/types'; +import { MikroOrmStorageConfig } from '@powersync/service-module-mikroorm-storage/types'; import { MongoStorageConfig } from '@powersync/service-module-mongodb-storage/types'; import { MongoConnectionConfig } from '@powersync/service-module-mongodb/types'; import { MSSQLConnectionConfig } from '@powersync/service-module-mssql/types'; @@ -28,7 +28,7 @@ const mergedDataSourceConfig = configFile.genericDataSourceConfig const mergedStorageConfig = configFile.GenericStorageConfig.or(PostgresStorageConfig) .or(MongoStorageConfig) - .or(MikroOrmSqliteStorageConfig); + .or(MikroOrmStorageConfig); const mergedConfig = t.object({ ...baseShape, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8ed3efa5..ca53802be 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -271,6 +271,9 @@ importers: '@mikro-orm/migrations': specifier: ^7.1.4 version: 7.1.4(@mikro-orm/core@7.1.4) + '@mikro-orm/mysql': + specifier: ^7.1.4 + version: 7.1.4(@mikro-orm/core@7.1.4)(@types/node@25.5.0) '@mikro-orm/sql': specifier: ^7.1.4 version: 7.1.4(@mikro-orm/core@7.1.4) @@ -1416,6 +1419,12 @@ packages: peerDependencies: '@mikro-orm/core': 7.1.4 + '@mikro-orm/mysql@7.1.4': + resolution: {integrity: sha512-Gck977Bp91KD9gB3TO0Trd46jiNw0fSM53kPOiX9WiBNoYdKIli6Yz8mAzYnd63dYb2q67jif5/gjZ2IMYlJIQ==} + engines: {node: '>= 22.17.0'} + peerDependencies: + '@mikro-orm/core': 7.1.4 + '@mikro-orm/sql@7.1.4': resolution: {integrity: sha512-YOie3z0Y24XlzWPUrATGJja5U6Ij81gji9yEDT6JkZxJH9uF1tbmZ2qX9WHectnMB5h8JHCic5AMvaflhBviNg==} engines: {node: '>= 22.17.0'} @@ -3274,6 +3283,9 @@ packages: long@5.2.3: resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + lossless-json@2.0.11: resolution: {integrity: sha512-BP0vn+NGYvzDielvBZaFain/wgeJ1hTvURCqtKvhr1SCPePdaaTanmmcplrHfEJSJOUql7hk4FHwToNJjWRY3g==} @@ -3288,6 +3300,10 @@ packages: resolution: {integrity: sha512-FbAj6lXil6t8z4z3j0E5mfRlPzxkySotzUHwRXjlpRh10vc6AI6WN62ehZj82VG7M20rqogJ0GLwar2Xa05a8Q==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lru.min@1.1.4: + resolution: {integrity: sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==} + engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lucia@3.2.2: resolution: {integrity: sha512-P1FlFBGCMPMXu+EGdVD9W4Mjm0DqsusmKgO7Xc33mI5X1bklmsQb0hfzPhXomQr9waWIBDsiOjvr1e6BTaUqpA==} deprecated: This package has been deprecated. Please see https://lucia-auth.com/lucia-v3/migrate. @@ -3413,10 +3429,20 @@ packages: resolution: {integrity: sha512-Qpu2ADfbKzyLdwC/5d4W7+5Yz7yBzCU05YWt5npWzACST37wJsB23wgOSo00qi043urkiRwXtEvJc9UnuLX/MQ==} engines: {node: '>= 8.0'} + mysql2@3.22.4: + resolution: {integrity: sha512-CtXYlmL7ZamiYKbmqkamQHWJROUHSfm+f3kByzGfknw7kW51mcB2ouMUqYq1XfYxbXmnWo6RhPydx6OCqdgcmQ==} + engines: {node: '>= 8.0'} + peerDependencies: + '@types/node': '>= 8' + named-placeholders@1.1.3: resolution: {integrity: sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==} engines: {node: '>=12.0.0'} + named-placeholders@1.1.6: + resolution: {integrity: sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==} + engines: {node: '>=8.0.0'} + nanoid@3.3.11: resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -3959,6 +3985,10 @@ packages: sprintf-js@1.1.3: resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + sql-escaper@1.3.3: + resolution: {integrity: sha512-BsTCV265VpTp8tm1wyIm1xqQCS+Q9NHx2Sr+WcnUrgLrQ6yiDIvHYJV5gHxsj1lMBy2zm5twLaZao8Jd+S8JJw==} + engines: {bun: '>=1.0.0', deno: '>=2.0.0', node: '>=12.0.0'} + sql-formatter@15.4.9: resolution: {integrity: sha512-5vmt2HlCAVozxsBZuXWkAki/KGawaK+b5GG5x+BtXOFVpN/8cqppblFUxHl4jxdA0cvo14lABhM+KBnrUapOlw==} hasBin: true @@ -4983,6 +5013,16 @@ snapshots: '@mikro-orm/core': 7.1.4 '@mikro-orm/sql': 7.1.4(@mikro-orm/core@7.1.4) + '@mikro-orm/mysql@7.1.4(@mikro-orm/core@7.1.4)(@types/node@25.5.0)': + dependencies: + '@mikro-orm/core': 7.1.4 + '@mikro-orm/sql': 7.1.4(@mikro-orm/core@7.1.4) + kysely: 0.29.2 + mysql2: 3.22.4(@types/node@25.5.0) + sqlstring: 2.3.3 + transitivePeerDependencies: + - '@types/node' + '@mikro-orm/sql@7.1.4(@mikro-orm/core@7.1.4)': dependencies: '@mikro-orm/core': 7.1.4 @@ -6831,6 +6871,8 @@ snapshots: long@5.2.3: {} + long@5.3.2: {} + lossless-json@2.0.11: {} lru-cache@10.4.3: {} @@ -6839,6 +6881,8 @@ snapshots: lru.min@1.1.1: {} + lru.min@1.1.4: {} + lucia@3.2.2: dependencies: '@oslojs/crypto': 1.0.1 @@ -6943,10 +6987,26 @@ snapshots: seq-queue: 0.0.5 sqlstring: 2.3.3 + mysql2@3.22.4(@types/node@25.5.0): + dependencies: + '@types/node': 25.5.0 + aws-ssl-profiles: 1.1.2 + denque: 2.1.0 + generate-function: 2.3.1 + iconv-lite: 0.7.2 + long: 5.3.2 + lru.min: 1.1.4 + named-placeholders: 1.1.6 + sql-escaper: 1.3.3 + named-placeholders@1.1.3: dependencies: lru-cache: 7.18.3 + named-placeholders@1.1.6: + dependencies: + lru.min: 1.1.1 + nanoid@3.3.11: {} napi-build-utils@2.0.0: {} @@ -7502,6 +7562,8 @@ snapshots: sprintf-js@1.1.3: {} + sql-escaper@1.3.3: {} + sql-formatter@15.4.9: dependencies: argparse: 2.0.1 diff --git a/service/src/util/modules.ts b/service/src/util/modules.ts index 5232dc05f..c85114fcd 100644 --- a/service/src/util/modules.ts +++ b/service/src/util/modules.ts @@ -13,6 +13,8 @@ export const DYNAMIC_MODULES: core.ModuleLoaders = { import('@powersync/service-module-mongodb-storage').then((module) => new module.MongoStorageModule()), 'mikroorm:sqlite': () => import('@powersync/service-module-mikroorm-storage').then((module) => new module.MikroOrmStorageModule()), + 'mikroorm:mysql': () => + import('@powersync/service-module-mikroorm-storage').then((module) => new module.MikroOrmStorageModule()), postgresql: () => import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule()) } From 1b10fdd22607fca5449e753ac6421a7c27ddc5e8 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 18 Jun 2026 17:09:12 +0200 Subject: [PATCH 05/11] add benchmarks and some basic performance improvements --- .../src/drivers/mysql/mysql-dialect.ts | 52 +-- .../src/drivers/sql/bucket-data-read.ts | 71 +++ .../src/drivers/sqlite/sqlite-dialect.ts | 39 +- .../test/src/storage_bench.test.ts | 40 ++ .../storage/implementation/MongoChecksums.ts | 15 +- .../implementation/v3/MongoChecksumsV3.ts | 21 +- .../v3/MongoSyncBucketStorageV3.ts | 28 +- .../v3/VersionedPowerSyncMongoV3.ts | 28 +- .../src/utils/test-utils.ts | 32 +- .../test/src/storage.test.ts | 41 +- .../test/src/storage_bench.test.ts | 35 ++ .../test/src/storage_bench.test.ts | 22 + .../benchmarks/register-storage-benchmarks.ts | 440 ++++++++++++++++++ .../src/tests/tests-index.ts | 1 + plans/sync-bucket-storage-benchmarks.md | 112 +++++ 15 files changed, 890 insertions(+), 87 deletions(-) create mode 100644 modules/module-mikroorm-storage/src/drivers/sql/bucket-data-read.ts create mode 100644 modules/module-mikroorm-storage/test/src/storage_bench.test.ts create mode 100644 modules/module-mongodb-storage/test/src/storage_bench.test.ts create mode 100644 modules/module-postgres-storage/test/src/storage_bench.test.ts create mode 100644 packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts create mode 100644 plans/sync-bucket-storage-benchmarks.md diff --git a/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts index 0ac570be4..587b7551c 100644 --- a/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts +++ b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts @@ -1,4 +1,3 @@ -import type { SqlEntityManager } from '@mikro-orm/sql'; import { BucketData, BucketDataSchema, @@ -17,6 +16,7 @@ import { } from '../../entities/entities-index.js'; import { InProcessMikroOrmCheckpointWatcher, MikroOrmStorageDialect } from '../../storage/MikroOrmStorageDialect.js'; import { MIKRO_ORM_MYSQL_STORAGE_TYPE } from '../../types/types.js'; +import { streamBucketDataRowsFromSql } from '../sql/bucket-data-read.js'; export const mysqlMikroOrmStorageDialect: MikroOrmStorageDialect = { type: MIKRO_ORM_MYSQL_STORAGE_TYPE, @@ -41,37 +41,27 @@ export const mysqlMikroOrmStorageDialect: MikroOrmStorageDialect = { return; } - const filters = options.dataBuckets.map((request) => ({ - bucketName: request.bucket, - opId: { $gt: request.start, $lte: options.checkpoint } - })); - const sqlEntityManager = options.em as SqlEntityManager; - const queryBuilder = sqlEntityManager - .createQueryBuilder(BucketData, 'bucket_data') - .select('*') - .where({ - groupId: options.groupId, - $or: filters - }) - .orderBy({ bucketName: 'ASC', opId: 'ASC' }) - .limit(options.limit); + yield* streamBucketDataRowsFromSql(options, (queryOptions) => { + const requestedRows = queryOptions.dataBuckets + .map((_, index) => `${index == 0 ? 'SELECT' : 'UNION ALL SELECT'} ? AS bucket_name, ? AS start_op_id`) + .join(' '); + const params = queryOptions.dataBuckets.flatMap((request) => [request.bucket, request.start]); - yield* streamQueryBuilder(queryBuilder.stream()); + return { + sql: ` + SELECT bucket_data.* + FROM (${requestedRows}) AS requested + JOIN bucket_data FORCE INDEX (bucket_data_bucket_op_index) + ON bucket_data.group_id = ? + AND bucket_data.bucket_name = requested.bucket_name + AND bucket_data.op_id > requested.start_op_id + AND bucket_data.op_id <= ? + ORDER BY bucket_data.bucket_name ASC, bucket_data.op_id ASC + LIMIT ? + `, + params: [...params, queryOptions.groupId, queryOptions.checkpoint, queryOptions.limit] + }; + }); }, createCheckpointWatcher: () => new InProcessMikroOrmCheckpointWatcher() }; - -async function* streamQueryBuilder(stream: AsyncIterableIterator): AsyncIterable { - const iterator = stream[Symbol.asyncIterator](); - try { - while (true) { - const result = await iterator.next(); - if (result.done) { - return; - } - yield result.value; - } - } finally { - await iterator.return?.(); - } -} diff --git a/modules/module-mikroorm-storage/src/drivers/sql/bucket-data-read.ts b/modules/module-mikroorm-storage/src/drivers/sql/bucket-data-read.ts new file mode 100644 index 000000000..4df2003c9 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sql/bucket-data-read.ts @@ -0,0 +1,71 @@ +import type { SqlEntityManager } from '@mikro-orm/sql'; +import type { BucketData } from '../../entities/entities-index.js'; +import type { MikroOrmBucketDataStreamOptions } from '../../storage/MikroOrmStorageDialect.js'; + +const REQUEST_BUCKET_CHUNK_SIZE = 400; + +export interface SqlBucketDataReadQueryOptions extends MikroOrmBucketDataStreamOptions { + readonly limit: number; +} + +export interface SqlBucketDataReadQuery { + readonly sql: string; + readonly params: unknown[]; +} + +export async function* streamBucketDataRowsFromSql( + options: MikroOrmBucketDataStreamOptions, + buildQuery: (options: SqlBucketDataReadQueryOptions) => SqlBucketDataReadQuery +): AsyncIterable { + const sqlEntityManager = options.em as SqlEntityManager; + const sortedBuckets = [...options.dataBuckets].sort((a, b) => a.bucket.localeCompare(b.bucket)); + let remainingLimit = options.limit; + + for (let offset = 0; offset < sortedBuckets.length && remainingLimit > 0; offset += REQUEST_BUCKET_CHUNK_SIZE) { + const dataBuckets = sortedBuckets.slice(offset, offset + REQUEST_BUCKET_CHUNK_SIZE); + const query = buildQuery({ + ...options, + dataBuckets, + limit: remainingLimit + }); + const rows = await sqlEntityManager.getConnection().execute(query.sql, query.params, 'all'); + + for (const row of rows) { + yield rawBucketDataRow(row); + } + + remainingLimit -= rows.length; + } +} + +interface RawBucketDataRow { + id: string; + group_id: number; + bucket_name: string; + op_id: bigint | number | string; + op: string; + source_table: string | null; + source_key: Uint8Array | null; + table_name: string | null; + row_id: string | null; + checksum: bigint | number | string; + data: string | null; + target_op: bigint | number | string | null; +} + +function rawBucketDataRow(row: RawBucketDataRow): BucketData { + return { + id: row.id, + groupId: row.group_id, + bucketName: row.bucket_name, + opId: BigInt(row.op_id), + op: row.op, + sourceTable: row.source_table, + sourceKey: row.source_key, + tableName: row.table_name, + rowId: row.row_id, + checksum: BigInt(row.checksum), + data: row.data, + targetOp: row.target_op == null ? null : BigInt(row.target_op) + }; +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts index 998c0dd3f..31bc76363 100644 --- a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts @@ -46,22 +46,33 @@ export const sqliteMikroOrmStorageDialect: MikroOrmStorageDialect = { // the query so single-process unified mode remains cooperative. await new Promise((resolve) => setImmediate(resolve)); - const filters = options.dataBuckets.map((request) => ({ - bucketName: request.bucket, - opId: { $gt: request.start, $lte: options.checkpoint } - })); const sqlEntityManager = options.em as SqlEntityManager; - const queryBuilder = sqlEntityManager - .createQueryBuilder(BucketData, 'bucket_data') - .select('*') - .where({ - groupId: options.groupId, - $or: filters - }) - .orderBy({ bucketName: 'ASC', opId: 'ASC' }) - .limit(options.limit); + const sortedBuckets = [...options.dataBuckets].sort((a, b) => a.bucket.localeCompare(b.bucket)); + let remainingLimit = options.limit; - yield* streamQueryBuilder(queryBuilder.stream()); + for (const request of sortedBuckets) { + if (remainingLimit <= 0) { + break; + } + + // Query each bucket as its own indexed range scan. In SQLite this was faster than joining a VALUES table for + // many buckets because it avoids cross-bucket planning and temporary sorting work. + const queryBuilder = sqlEntityManager + .createQueryBuilder(BucketData, 'bucket_data') + .select('*') + .where({ + groupId: options.groupId, + bucketName: request.bucket, + opId: { $gt: request.start, $lte: options.checkpoint } + }) + .orderBy({ opId: 'ASC' }) + .limit(remainingLimit); + + for await (const row of streamQueryBuilder(queryBuilder.stream())) { + yield row; + remainingLimit--; + } + } }, createCheckpointWatcher: () => new InProcessMikroOrmCheckpointWatcher() }; diff --git a/modules/module-mikroorm-storage/test/src/storage_bench.test.ts b/modules/module-mikroorm-storage/test/src/storage_bench.test.ts new file mode 100644 index 000000000..e4d91cb6a --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/storage_bench.test.ts @@ -0,0 +1,40 @@ +import type { StorageBenchmarkResult } from '@powersync/service-core-tests'; +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { env } from './env.js'; +import { MIKRO_ORM_MYSQL_STORAGE_FACTORY, MIKRO_ORM_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +const results: StorageBenchmarkResult[] = []; +register.registerStorageBenchmarkSummary(results); + +describe.sequential('MikroORM SQLite Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion }, + { + storageName: 'mikroorm:sqlite', + storageVersion, + results + } + ); + }); + } +}); + +describe + .skipIf(!env.MIKROORM_MYSQL_STORAGE_TEST_URI) + .sequential('MikroORM MySQL Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion }, + { + storageName: 'mikroorm:mysql', + storageVersion, + results + } + ); + }); + } + }); diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts index 747a78d77..f1957f04d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoChecksums.ts @@ -185,7 +185,11 @@ export abstract class MongoChecksums { >( batch: TRequest[], collection: lib_mongo.mongo.Collection, - createFilter: (request: TRequest) => any + createFilter: (request: TRequest) => any, + options: { + sort?: Record; + hint?: string; + } = {} ): Promise { const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT; @@ -223,7 +227,7 @@ export abstract class MongoChecksums { } }, // sort and limit _before_ grouping - { $sort: { _id: 1 } }, + { $sort: options.sort ?? { _id: 1 } }, { $limit: batchLimit }, { $group: { @@ -242,7 +246,12 @@ export abstract class MongoChecksums { // This is important to identify which buckets we have partial data for. { $sort: { _id: 1 } } ], - { session: undefined, readConcern: 'snapshot', maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS } + { + session: undefined, + readConcern: 'snapshot', + maxTimeMS: lib_mongo.MONGO_CHECKSUM_TIMEOUT_MS, + ...(options.hint != null ? { hint: options.hint } : {}) + } ) .toArray() .catch((e) => { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts index 88e0b42fa..2598ca5fe 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoChecksumsV3.ts @@ -13,7 +13,7 @@ import { MongoChecksumOptions, MongoChecksums } from '../MongoChecksums.js'; -import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; +import { BUCKET_DATA_BUCKET_OP_INDEX, VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoChecksumsV3 extends MongoChecksums { declare protected readonly db: VersionedPowerSyncMongoV3; @@ -47,7 +47,11 @@ export class MongoChecksumsV3 extends MongoChecksums { const groupResults = await this.computePartialChecksumsForCollection( requests, this.db.bucketDataV3(this.group_id, definitionId), - createV3BucketFilter + createV3BucketFilter, + { + sort: { '_id.b': 1, '_id.o': 1 }, + hint: BUCKET_DATA_BUCKET_OP_INDEX + } ); for (const checksum of groupResults.values()) { results.set(checksum.bucket, checksum); @@ -106,15 +110,10 @@ export class MongoChecksumsV3 extends MongoChecksums { function createV3BucketFilter(request: Pick) { return { - _id: { - $gt: { - b: request.bucket, - o: request.start ?? new bson.MinKey() - }, - $lte: { - b: request.bucket, - o: request.end - } + '_id.b': request.bucket, + '_id.o': { + $gt: request.start ?? new bson.MinKey(), + $lte: request.end } }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 7a51ee339..53bab63d3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -39,7 +39,7 @@ import { MongoChecksumsV3 } from './MongoChecksumsV3.js'; import { MongoCompactorV3 } from './MongoCompactorV3.js'; import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js'; import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js'; -import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; +import { BUCKET_DATA_BUCKET_OP_INDEX, VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { // Declare types to be more specific @@ -105,15 +105,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { protected async initializeVersionStorage(): Promise { const mapping = this.mapping; for (let source of mapping.allBucketDefinitionIds()) { - const collection = this.db.bucketDataV3(this.replicationStreamId, source).collectionName; - await this.db.db - .createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }) - .catch((error) => { - if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') { - return; - } - throw error; - }); + await this.db.initializeBucketDataCollection(this.replicationStreamId, source); } for (let indexId of mapping.allParameterIndexIds()) { await this.db.parameterIndexV3(this.replicationStreamId, indexId).createIndex( @@ -456,15 +448,10 @@ export async function* getBucketDataBatchV3( const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1; const bucketMap = new Map(requests.map((request) => [request.bucket, request.start])); const filters: mongo.Filter[] = Array.from(bucketMap.entries()).map(([bucket, start]) => ({ - _id: { - $gt: { - b: bucket, - o: start - }, - $lte: { - b: bucket, - o: end as any - } + '_id.b': bucket, + '_id.o': { + $gt: start, + $lte: end } })); @@ -474,7 +461,8 @@ export async function* getBucketDataBatchV3( }, { session: undefined, - sort: { _id: 1 }, + sort: { '_id.b': 1, '_id.o': 1 }, + hint: BUCKET_DATA_BUCKET_OP_INDEX, limit: remainingLimit, batchSize: remainingLimit + 1, raw: true, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts index 68b407422..998699cb0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -1,4 +1,4 @@ -import { mongo } from '@powersync/lib-service-mongodb'; +import { isMongoServerError, mongo } from '@powersync/lib-service-mongodb'; import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules'; import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js'; import { CommonSourceTableDocument } from '../models.js'; @@ -11,6 +11,8 @@ import { SyncConfigDefinition } from './models.js'; +export const BUCKET_DATA_BUCKET_OP_INDEX = 'bucket_op'; + export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { get syncConfigDefinitions() { return this.db.collection('sync_config'); @@ -97,6 +99,30 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { return this.db.collection(`bucket_data_${replicationStreamId}_${definitionId}`); } + async initializeBucketDataCollection(replicationStreamId: number, definitionId: BucketDefinitionId) { + const collection = this.bucketDataV3(replicationStreamId, definitionId); + await this.db + .createCollection(collection.collectionName, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } }) + .catch((error) => { + if (isMongoServerError(error) && error.codeName === 'NamespaceExists') { + return; + } + throw error; + }); + + await collection.createIndex( + { + '_id.b': 1, + '_id.o': 1, + checksum: 1, + op: 1 + }, + { + name: BUCKET_DATA_BUCKET_OP_INDEX + } + ); + } + listBucketDataCollectionsV3(replicationStreamId: number) { return this.upstream.listBucketDataCollectionsV3(replicationStreamId); } diff --git a/modules/module-mongodb-storage/src/utils/test-utils.ts b/modules/module-mongodb-storage/src/utils/test-utils.ts index af8755490..deab8595e 100644 --- a/modules/module-mongodb-storage/src/utils/test-utils.ts +++ b/modules/module-mongodb-storage/src/utils/test-utils.ts @@ -1,5 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { TestStorageOptions } from '@powersync/service-core'; +import { framework, PowerSyncMigrationManager, TestStorageOptions } from '@powersync/service-core'; +import { MongoMigrationAgent } from '../migrations/MongoMigrationAgent.js'; import { MongoBucketStorage, MongoBucketStorageOptions } from '../storage/MongoBucketStorage.js'; import { MongoReportStorage } from '../storage/MongoReportStorage.js'; import { PowerSyncMongo } from '../storage/implementation/db.js'; @@ -7,13 +8,19 @@ import { PowerSyncMongo } from '../storage/implementation/db.js'; export type MongoTestStorageOptions = { url: string; isCI: boolean; + clientOptions?: mongo.MongoClientOptions; + runMigrations?: boolean; internalOptions?: MongoBucketStorageOptions; }; export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) { return { factory: async (options?: TestStorageOptions) => { - const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); + if (factoryOptions.runMigrations) { + await runMongoMigrations(factoryOptions.url); + } + + const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.clientOptions); // None of the tests insert data into this collection, so it was never created if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) { @@ -35,7 +42,11 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) { return async (options?: TestStorageOptions) => { - const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); + if (factoryOptions.runMigrations) { + await runMongoMigrations(factoryOptions.url); + } + + const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.clientOptions); await db.createConnectionReportingCollection(); @@ -47,13 +58,24 @@ export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTest }; } -export const connectMongoForTests = (url: string, isCI: boolean) => { +export const connectMongoForTests = (url: string, isCI: boolean, options: mongo.MongoClientOptions = {}) => { // Short timeout for tests, to fail fast when the server is not available. // Slightly longer timeouts for CI, to avoid arbitrary test failures const client = new mongo.MongoClient(url, { connectTimeoutMS: isCI ? 15_000 : 5_000, socketTimeoutMS: isCI ? 15_000 : 5_000, - serverSelectionTimeoutMS: isCI ? 15_000 : 2_500 + serverSelectionTimeoutMS: isCI ? 15_000 : 2_500, + ...options }); return new PowerSyncMongo(client); }; + +async function runMongoMigrations(url: string) { + await using migrationManager: PowerSyncMigrationManager = new framework.migrations.MigrationManager(); + await using migrationAgent = new MongoMigrationAgent({ type: 'mongodb', uri: url }); + + migrationManager.registerMigrationAgent(migrationAgent); + await migrationManager.migrate({ + direction: framework.migrations.Direction.Up + }); +} diff --git a/modules/module-mongodb-storage/test/src/storage.test.ts b/modules/module-mongodb-storage/test/src/storage.test.ts index 16bfa9690..a2de94e6b 100644 --- a/modules/module-mongodb-storage/test/src/storage.test.ts +++ b/modules/module-mongodb-storage/test/src/storage.test.ts @@ -1,6 +1,8 @@ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js'; -import { register } from '@powersync/service-core-tests'; -import { describe } from 'vitest'; +import { updateSyncRulesFromYaml } from '@powersync/service-core'; +import { register, test_utils } from '@powersync/service-core-tests'; +import { describe, expect, test } from 'vitest'; +import type { MongoBucketStorage } from '../../src/index.js'; import { env } from './env.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -17,6 +19,41 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { describe('Sync Bucket Validation', register.registerBucketValidationTests); +test('Mongo v3 bucket data collections use the optimized bucket/op index', async () => { + await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); + const replicationStream = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +bucket_definitions: + global: + data: + - SELECT * FROM test +`, + { storageVersion: 3 } + ) + ); + + const bucketStorage = factory.getInstance(replicationStream); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + void writer; + + const mongoFactory = factory as MongoBucketStorage; + const collections = await mongoFactory.db.listBucketDataCollectionsV3(Number(replicationStream.replicationStreamId)); + expect(collections).toHaveLength(1); + const indexes = await collections[0].indexes(); + expect(indexes).toContainEqual( + expect.objectContaining({ + name: 'bucket_op', + key: { + '_id.b': 1, + '_id.o': 1, + checksum: 1, + op: 1 + } + }) + ); +}); + describe('Mongo Sync Bucket Storage - split operations', () => register.registerDataStorageDataTests( mongoTestStorageFactoryGenerator({ diff --git a/modules/module-mongodb-storage/test/src/storage_bench.test.ts b/modules/module-mongodb-storage/test/src/storage_bench.test.ts new file mode 100644 index 000000000..f62cf7cef --- /dev/null +++ b/modules/module-mongodb-storage/test/src/storage_bench.test.ts @@ -0,0 +1,35 @@ +import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js'; +import type { StorageBenchmarkResult } from '@powersync/service-core-tests'; +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { env } from './env.js'; +import { TEST_STORAGE_VERSIONS } from './util.js'; + +const MONGO_BENCHMARK_STORAGE_FACTORY = mongoTestStorageFactoryGenerator({ + url: env.MONGO_TEST_URL, + isCI: env.CI, + runMigrations: true, + clientOptions: { + connectTimeoutMS: 60_000, + socketTimeoutMS: 1_200_000, + serverSelectionTimeoutMS: 60_000 + } +}); + +const results: StorageBenchmarkResult[] = []; +register.registerStorageBenchmarkSummary(results); + +describe.sequential('MongoDB Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...MONGO_BENCHMARK_STORAGE_FACTORY, storageVersion }, + { + storageName: 'mongodb', + storageVersion, + results + } + ); + }); + } +}); diff --git a/modules/module-postgres-storage/test/src/storage_bench.test.ts b/modules/module-postgres-storage/test/src/storage_bench.test.ts new file mode 100644 index 000000000..341f91469 --- /dev/null +++ b/modules/module-postgres-storage/test/src/storage_bench.test.ts @@ -0,0 +1,22 @@ +import type { StorageBenchmarkResult } from '@powersync/service-core-tests'; +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +const results: StorageBenchmarkResult[] = []; +register.registerStorageBenchmarkSummary(results); + +describe.sequential('Postgres Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...POSTGRES_STORAGE_FACTORY, storageVersion }, + { + storageName: 'postgresql', + storageVersion, + results + } + ); + }); + } +}); diff --git a/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts b/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts new file mode 100644 index 000000000..fe6f71df0 --- /dev/null +++ b/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts @@ -0,0 +1,440 @@ +import { createCoreAPIMetrics, JwtPayload, storage, sync, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { performance } from 'node:perf_hooks'; +import { afterAll, test } from 'vitest'; +import { + BATCH_OPTIONS, + deploySyncRules, + METRICS_HELPER, + PARSE_OPTIONS, + resolveTestTable +} from '../test-utils/test-utils-index.js'; + +const BENCHMARK_SYNC_RULES = ` +bucket_definitions: + by_list: + parameters: + - SELECT id AS list_id FROM lists + data: + - SELECT id, list_id, owner_id, description, completed FROM todos WHERE list_id = bucket.list_id + by_user: + parameters: + - SELECT owner_id AS owner_id FROM lists + data: + - SELECT id, list_id, owner_id, description, completed FROM todos WHERE owner_id = bucket.owner_id +`; + +export interface StorageBenchmarkScenario { + name: string; + /** Number of todo rows to persist and expose through data buckets. */ + todo_row_count: number; + /** Number of list rows to persist and expose as parameter buckets. */ + list_row_count: number; + /** Number of distinct users spread over list and todo rows. */ + user_count: number; + /** Flush pending source rows after this many saves. */ + flush_every: number; + /** Maximum expected number of resolved buckets for this scenario. */ + max_bucket_count?: number; + /** Optional per-scenario Vitest timeout. */ + timeout_ms?: number; +} + +export interface RegisterStorageBenchmarkOptions { + storageName: string; + storageVersion?: number; + scenarios?: StorageBenchmarkScenario[]; + results?: StorageBenchmarkResult[]; + timeout_ms?: number; + progress_interval_ms?: number; +} + +export interface StorageBenchmarkResult { + storage: string; + version: number | null; + scenario: string; + source_rows: number; + buckets: number; + write_ms: number; + write_rows_per_second: number; + sync_drain_ms: number; + ops: number; + mebibytes_per_second: number; +} + +export const DEFAULT_STORAGE_BENCHMARK_SCENARIOS: StorageBenchmarkScenario[] = [ + { + name: 'small', + todo_row_count: 1_000, + list_row_count: 100, + user_count: 100, + flush_every: 1_000, + max_bucket_count: 1_000, + timeout_ms: 120_000 + }, + { + name: 'medium', + todo_row_count: 10_000, + list_row_count: 500, + user_count: 500, + flush_every: 1_000, + max_bucket_count: 1_000, + timeout_ms: 300_000 + }, + { + name: 'large', + todo_row_count: 100_000, + list_row_count: 500, + user_count: 500, + flush_every: 1_000, + max_bucket_count: 1_000, + timeout_ms: 1_200_000 + } +]; + +export function registerStorageBenchmarks( + configOrFactory: storage.TestStorageConfig | storage.TestStorageFactory, + options: RegisterStorageBenchmarkOptions +) { + const config: storage.TestStorageConfig = + typeof configOrFactory == 'function' ? { factory: configOrFactory, tableIdStrings: true } : configOrFactory; + const storageVersion = options.storageVersion ?? config.storageVersion; + const scenarios = options.scenarios ?? DEFAULT_STORAGE_BENCHMARK_SCENARIOS; + + createCoreAPIMetrics(METRICS_HELPER.metricsEngine); + + for (const scenario of scenarios) { + test( + `storage benchmark - ${scenario.name}`, + async () => { + const result = await runStorageBenchmark( + config, + { + ...options, + storageVersion + }, + scenario + ); + + options.results?.push(result); + console.log(formatStorageBenchmarkResultLine(result)); + }, + scenario.timeout_ms ?? options.timeout_ms ?? 1_200_000 + ); + } +} + +export function registerStorageBenchmarkSummary( + results: StorageBenchmarkResult[], + title = 'Storage benchmark results' +) { + afterAll(() => { + if (results.length == 0) { + return; + } + + console.log(`\n${title}\n${formatStorageBenchmarkResults(results)}`); + }); +} + +export function formatStorageBenchmarkResults(results: StorageBenchmarkResult[]) { + const rows = [ + '| Storage | Version | Scenario | Source Rows | Buckets | Write ms | Write rows/s | Sync drain ms | Ops | MiB/s |', + '| --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |' + ]; + + for (const result of results) { + rows.push( + `| ${result.storage} | ${result.version ?? ''} | ${result.scenario} | ${result.source_rows} | ${ + result.buckets + } | ${formatNumber(result.write_ms)} | ${formatNumber(result.write_rows_per_second)} | ${formatNumber( + result.sync_drain_ms + )} | ${result.ops} | ${formatNumber(result.mebibytes_per_second)} |` + ); + } + + return rows.join('\n'); +} + +async function runStorageBenchmark( + config: storage.TestStorageConfig, + options: Required> & + Omit, + scenario: StorageBenchmarkScenario +): Promise { + validateScenario(scenario); + + await using factory = await config.factory(); + const syncRules = await deploySyncRules( + factory, + updateSyncRulesFromYaml(BENCHMARK_SYNC_RULES, { + storageVersion: options.storageVersion + }) + ); + + const bucketStorage = factory.getInstance(syncRules.stream); + const label = benchmarkLabel(options.storageName, options.storageVersion, scenario.name); + const sourceRows = scenario.todo_row_count + scenario.list_row_count; + + const writeStarted = performance.now(); + await persistBenchmarkRows({ + config, + bucketStorage, + scenario, + label, + progressIntervalMs: options.progress_interval_ms ?? 5_000 + }); + const writeMs = performance.now() - writeStarted; + + const drain = await drainSyncStream({ + bucketStorage, + label, + scenario, + progressIntervalMs: options.progress_interval_ms ?? 5_000 + }); + + return { + storage: options.storageName, + version: options.storageVersion ?? null, + scenario: scenario.name, + source_rows: sourceRows, + buckets: drain.buckets, + write_ms: writeMs, + write_rows_per_second: sourceRows / (writeMs / 1_000), + sync_drain_ms: drain.elapsedMs, + ops: drain.ops, + mebibytes_per_second: drain.bytes / 1024 / 1024 / (drain.elapsedMs / 1_000) + }; +} + +async function persistBenchmarkRows(options: { + config: storage.TestStorageConfig; + bucketStorage: storage.SyncRulesBucketStorage; + scenario: StorageBenchmarkScenario; + label: string; + progressIntervalMs: number; +}) { + const { config, bucketStorage, scenario, label } = options; + const sourceRows = scenario.todo_row_count + scenario.list_row_count; + const started = performance.now(); + let lastProgress = started; + let savedRows = 0; + let pendingRows = 0; + + await using writer = await bucketStorage.createWriter(BATCH_OPTIONS); + const listsTable = await resolveTestTable(writer, 'lists', ['id'], config, 1); + const todosTable = await resolveTestTable(writer, 'todos', ['id'], config, 2); + + await writer.markAllSnapshotDone('0/1'); + + async function saveRow(row: storage.SaveOptions) { + await writer.save(row); + savedRows++; + pendingRows++; + + if (pendingRows >= scenario.flush_every) { + await writer.flush(); + pendingRows = 0; + maybeLogProgress(); + } + } + + function maybeLogProgress(force = false) { + const now = performance.now(); + if (force || now - lastProgress >= options.progressIntervalMs) { + lastProgress = now; + console.log( + `[${label}] write progress source_rows=${savedRows}/${sourceRows} elapsed_ms=${formatNumber(now - started)}` + ); + } + } + + for (let i = 0; i < scenario.list_row_count; i++) { + const listId = listRowId(i); + await saveRow({ + sourceTable: listsTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: listId, + owner_id: userId(i % scenario.user_count), + name: `List ${i}` + }, + afterReplicaId: listId + }); + } + + for (let i = 0; i < scenario.todo_row_count; i++) { + const listIndex = i % scenario.list_row_count; + const todoId = todoRowId(i); + await saveRow({ + sourceTable: todosTable, + tag: storage.SaveOperationTag.INSERT, + after: { + id: todoId, + list_id: listRowId(listIndex), + owner_id: userId(listIndex % scenario.user_count), + description: `Todo ${i} for ${listRowId(listIndex)}`, + completed: i % 3 == 0 ? 1 : 0 + }, + afterReplicaId: todoId + }); + } + + maybeLogProgress(true); + await writer.commit('1/1'); +} + +async function drainSyncStream(options: { + bucketStorage: storage.SyncRulesBucketStorage; + label: string; + scenario: StorageBenchmarkScenario; + progressIntervalMs: number; +}) { + const { bucketStorage, label, scenario, progressIntervalMs } = options; + const started = performance.now(); + const tracker = new sync.RequestTracker(METRICS_HELPER.metricsEngine); + const controller = new AbortController(); + const maxBucketCount = expectedBucketCount(scenario); + const syncContext = new sync.SyncContext({ + maxBuckets: maxBucketCount + 10, + maxParameterQueryResults: maxBucketCount + 10, + maxDataFetchConcurrency: 2 + }); + + let lines = 0; + let ops = 0; + let bytes = 0; + let buckets = 0; + let completed = false; + + const heartbeat = setInterval(() => { + console.log( + `[${label}] sync stream heartbeat lines=${lines} ops=${ops} bytes=${bytes} elapsed_ms=${formatNumber( + performance.now() - started + )}` + ); + }, progressIntervalMs); + heartbeat.unref?.(); + + try { + const stream = sync.streamResponse({ + syncContext, + bucketStorage, + syncRules: bucketStorage.getParsedSyncRules(PARSE_OPTIONS), + params: { + buckets: [], + include_checksum: true, + raw_data: true + }, + tracker, + token: new JwtPayload({ sub: 'user_00000000', exp: Date.now() / 1_000 + 3_600 }), + isEncodingAsBson: false, + signal: controller.signal + }); + + for await (const rawLine of stream) { + if (rawLine == null) { + continue; + } + + lines++; + const line = parseSyncLine(rawLine); + bytes += Buffer.byteLength(typeof rawLine == 'string' ? rawLine : JSONBig.stringify(rawLine)); + + if (line != null && 'checkpoint' in line) { + buckets = line.checkpoint.buckets.length; + } else if (line != null && 'checkpoint_diff' in line) { + buckets = Math.max(buckets, line.checkpoint_diff.updated_buckets.length); + } else if (line != null && 'data' in line) { + ops += line.data.data.length; + } else if (line != null && 'checkpoint_complete' in line) { + completed = true; + break; + } + } + } finally { + controller.abort(); + clearInterval(heartbeat); + } + + if (!completed) { + throw new Error(`[${label}] sync stream ended before checkpoint_complete`); + } + + return { + buckets, + ops, + bytes, + elapsedMs: performance.now() - started + }; +} + +function parseSyncLine(line: string | object) { + if (typeof line != 'string') { + return line as any; + } + + try { + return JSONBig.parse(line) as any; + } catch { + return null; + } +} + +function validateScenario(scenario: StorageBenchmarkScenario) { + if (scenario.todo_row_count < 1) { + throw new Error(`Benchmark scenario ${scenario.name} must have at least one todo row`); + } + if (scenario.list_row_count < 1) { + throw new Error(`Benchmark scenario ${scenario.name} must have at least one list row`); + } + if (scenario.user_count < 1) { + throw new Error(`Benchmark scenario ${scenario.name} must have at least one user`); + } + if (scenario.flush_every < 1) { + throw new Error(`Benchmark scenario ${scenario.name} must flush at least every one row`); + } + + const maxBucketCount = scenario.max_bucket_count ?? 1_000; + const buckets = expectedBucketCount(scenario); + if (buckets > maxBucketCount) { + throw new Error( + `Benchmark scenario ${scenario.name} creates ${buckets} buckets, exceeding max_bucket_count=${maxBucketCount}` + ); + } +} + +function expectedBucketCount(scenario: StorageBenchmarkScenario) { + return scenario.list_row_count + Math.min(scenario.list_row_count, scenario.user_count); +} + +function benchmarkLabel(storageName: string, storageVersion: number | undefined, scenarioName: string) { + return `${storageName}/v${storageVersion ?? 'default'}/${scenarioName}`; +} + +function listRowId(index: number) { + return `list_${String(index).padStart(8, '0')}`; +} + +function todoRowId(index: number) { + return `todo_${String(index).padStart(8, '0')}`; +} + +function userId(index: number) { + return `user_${String(index).padStart(8, '0')}`; +} + +function formatStorageBenchmarkResultLine(result: StorageBenchmarkResult) { + return `[${result.storage}/v${result.version ?? 'default'}/${result.scenario}] result write_ms=${formatNumber( + result.write_ms + )} write_rows_per_second=${formatNumber(result.write_rows_per_second)} sync_drain_ms=${formatNumber( + result.sync_drain_ms + )} ops=${result.ops} buckets=${result.buckets} mebibytes_per_second=${formatNumber(result.mebibytes_per_second)}`; +} + +function formatNumber(value: number) { + if (Number.isInteger(value)) { + return String(value); + } + return value.toFixed(value >= 100 ? 0 : 2); +} diff --git a/packages/service-core-tests/src/tests/tests-index.ts b/packages/service-core-tests/src/tests/tests-index.ts index a40468a32..2b1e28aca 100644 --- a/packages/service-core-tests/src/tests/tests-index.ts +++ b/packages/service-core-tests/src/tests/tests-index.ts @@ -1,3 +1,4 @@ +export * from '../benchmarks/register-storage-benchmarks.js'; export * from './register-bucket-validation-tests.js'; export * from './register-compacting-tests.js'; export * from './register-data-storage-checkpoint-tests.js'; diff --git a/plans/sync-bucket-storage-benchmarks.md b/plans/sync-bucket-storage-benchmarks.md new file mode 100644 index 000000000..cebfefc20 --- /dev/null +++ b/plans/sync-bucket-storage-benchmarks.md @@ -0,0 +1,112 @@ +# Sync Bucket Storage Benchmark Suite Plan + +## Goal + +Add a reusable benchmark suite for comparing PowerSync sync bucket storage implementations. The suite should measure the two storage paths that matter most for sync throughput: + +- Persisting a synthetic incoming dataset that touches both parameter lookups and bucket data writes. +- Draining all bucket data through the real sync stream API until `checkpoint_complete`. + +The benchmark harness belongs in `packages/service-core-tests` so every storage module can register the same scenarios without duplicating data generation or stream-drain logic. + +## Benchmark Model + +Use a source-DB-free synthetic dataset written through the storage writer API: + +- `lists` rows are parameter rows. +- `todos` rows are data rows. +- One bucket definition groups todos by list. +- One bucket definition groups todos by user. + +This makes each todo row appear in two bucket families while list rows drive the dynamic parameter hot path. + +Scenario fields are explicit test parameters, not environment variables: + +```ts +interface StorageBenchmarkScenario { + name: string; + todo_row_count: number; + list_row_count: number; + user_count: number; + flush_every: number; + max_bucket_count?: number; + timeout_ms?: number; +} +``` + +Default scenarios: + +| Scenario | Todo Rows | List Rows | Users | Max Buckets | +| -------- | --------: | --------: | ----: | ----------: | +| small | 1,000 | 100 | 100 | 1,000 | +| medium | 10,000 | 500 | 500 | 1,000 | +| large | 100,000 | 500 | 500 | 1,000 | + +The number of rows in buckets may grow large, but the total resolved bucket count must stay capped at 1,000 by default. All registered scenarios always run both the write phase and the sync stream drain phase. + +## Harness Requirements + +- Export `registerStorageBenchmarks()` from `@powersync/service-core-tests`. +- Export a benchmark summary printer that emits a markdown table after the suite. +- Accept a normal `TestStorageConfig` or `TestStorageFactory`. +- Accept storage name, storage version, scenario list, timeout, progress interval, and an optional shared result array. +- Write rows with `createWriter()` and flush every `flush_every` source rows. +- Commit once after all rows are written so the drain measures a single checkpoint. +- Drain via `sync.streamResponse()` with `raw_data: true` and stop only when `checkpoint_complete` is received. +- Do not benchmark a separate direct `getBucketDataBatch()` drain; the sync stream drain is the public path under test. +- Emit heartbeat progress logs during long sync drains showing lines, ops, bytes, and elapsed milliseconds. +- Emit write progress logs during long writes. + +## Storage Module Registration + +Each storage module should add a `test/src/storage_bench.test.ts` file and register all supported storage versions: + +- `modules/module-postgres-storage` +- `modules/module-mongodb-storage` +- `modules/module-mikroorm-storage` for SQLite and MySQL + +Storage setup must include migrations or schema setup before benchmark timing so indexes are present. Postgres and MikroORM use their existing test factory migration/schema paths. MongoDB benchmark factories should explicitly run migrations and use longer client socket timeouts so large drains are not killed by the normal fast-fail test timeout settings. + +## Sample Commands + +Run a single storage benchmark file with Vitest's `--run` mode: + +```sh +pnpm --filter @powersync/service-module-postgres-storage test test/src/storage_bench.test.ts --run +``` + +```sh +pnpm --filter @powersync/service-module-mongodb-storage test test/src/storage_bench.test.ts --run +``` + +```sh +pnpm --filter @powersync/service-module-mikroorm-storage test test/src/storage_bench.test.ts --run -t "MikroORM SQLite" +``` + +```sh +MIKROORM_MYSQL_STORAGE_TEST_URI="mysql://repl_user:good_password@localhost:3306/powersync" \ + corepack pnpm --filter @powersync/service-module-mikroorm-storage test test/src/storage_bench.test.ts --run -t "MikroORM MySQL" +``` + +The storage modules use their normal test database environment variables when supplied: + +- `PG_STORAGE_TEST_URL` for Postgres storage benchmarks. +- `MONGO_TEST_URL` for MongoDB storage benchmarks. +- `MIKROORM_MYSQL_STORAGE_TEST_URI` for MikroORM MySQL storage benchmarks. + +## Output + +The summary table should include: + +- Storage +- Version +- Scenario +- Source Rows +- Buckets +- Write ms +- Write rows/s +- Sync drain ms +- Ops +- MiB/s + +The output is intentionally plain markdown so benchmark results can be pasted into issues, PRs, or follow-up analysis notes. From 62c87fb5393e70ddd5626795155524de0cae25f4 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jul 2026 09:04:56 +0200 Subject: [PATCH 06/11] wip: drizzle support --- .../drizzle.sqlite.config.ts | 7 + modules/module-drizzle-storage/package.json | 51 ++ .../sqlite/SqliteDrizzleStorageFactory.ts | 17 + .../sqlite/SqliteMigrationLockManager.ts | 76 +++ .../src/drivers/sqlite/column-types.ts | 55 ++ .../src/drivers/sqlite/schema.ts | 137 +++++ .../src/drivers/sqlite/sqlite-config.ts | 66 ++ .../src/drivers/sqlite/sqlite-dialect.ts | 84 +++ modules/module-drizzle-storage/src/index.ts | 10 + .../src/migrations/DrizzleMigrationAgent.ts | 58 ++ .../src/migrations/NoOpMigrationStore.ts | 9 + .../migrations/sqlite/0000_real_harpoon.sql | 89 +++ .../migrations/sqlite/meta/0000_snapshot.json | 555 +++++++++++++++++ .../src/migrations/sqlite/meta/_journal.json | 13 + .../src/module/DrizzleStorageModule.ts | 21 + .../storage/DrizzleBucketStorageFactory.ts | 239 ++++++++ .../DrizzlePersistedReplicationStream.ts | 75 +++ .../src/storage/DrizzleReportStorage.ts | 34 ++ .../src/storage/DrizzleStorageDialect.ts | 55 ++ .../src/storage/DrizzleStorageProvider.ts | 67 ++ .../src/storage/DrizzleSyncRulesStorage.ts | 573 ++++++++++++++++++ .../src/storage/storage-index.ts | 5 + .../src/storage/unsupported.ts | 3 + .../src/types/records.ts | 98 +++ .../module-drizzle-storage/src/types/types.ts | 37 ++ .../test/src/migrations.test.ts | 81 +++ .../module-drizzle-storage/test/src/setup.ts | 11 + .../module-drizzle-storage/test/tsconfig.json | 11 + modules/module-drizzle-storage/tsconfig.json | 17 + .../module-drizzle-storage/vitest.config.ts | 7 + packages/schema/package.json | 1 + packages/schema/tsconfig.json | 3 + service/package.json | 1 + service/src/util/modules.ts | 2 + service/tsconfig.json | 3 + tsconfig.json | 6 + 36 files changed, 2577 insertions(+) create mode 100644 modules/module-drizzle-storage/drizzle.sqlite.config.ts create mode 100644 modules/module-drizzle-storage/package.json create mode 100644 modules/module-drizzle-storage/src/drivers/sqlite/SqliteDrizzleStorageFactory.ts create mode 100644 modules/module-drizzle-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts create mode 100644 modules/module-drizzle-storage/src/drivers/sqlite/column-types.ts create mode 100644 modules/module-drizzle-storage/src/drivers/sqlite/schema.ts create mode 100644 modules/module-drizzle-storage/src/drivers/sqlite/sqlite-config.ts create mode 100644 modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts create mode 100644 modules/module-drizzle-storage/src/index.ts create mode 100644 modules/module-drizzle-storage/src/migrations/DrizzleMigrationAgent.ts create mode 100644 modules/module-drizzle-storage/src/migrations/NoOpMigrationStore.ts create mode 100644 modules/module-drizzle-storage/src/migrations/sqlite/0000_real_harpoon.sql create mode 100644 modules/module-drizzle-storage/src/migrations/sqlite/meta/0000_snapshot.json create mode 100644 modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json create mode 100644 modules/module-drizzle-storage/src/module/DrizzleStorageModule.ts create mode 100644 modules/module-drizzle-storage/src/storage/DrizzleBucketStorageFactory.ts create mode 100644 modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts create mode 100644 modules/module-drizzle-storage/src/storage/DrizzleReportStorage.ts create mode 100644 modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts create mode 100644 modules/module-drizzle-storage/src/storage/DrizzleStorageProvider.ts create mode 100644 modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts create mode 100644 modules/module-drizzle-storage/src/storage/storage-index.ts create mode 100644 modules/module-drizzle-storage/src/storage/unsupported.ts create mode 100644 modules/module-drizzle-storage/src/types/records.ts create mode 100644 modules/module-drizzle-storage/src/types/types.ts create mode 100644 modules/module-drizzle-storage/test/src/migrations.test.ts create mode 100644 modules/module-drizzle-storage/test/src/setup.ts create mode 100644 modules/module-drizzle-storage/test/tsconfig.json create mode 100644 modules/module-drizzle-storage/tsconfig.json create mode 100644 modules/module-drizzle-storage/vitest.config.ts diff --git a/modules/module-drizzle-storage/drizzle.sqlite.config.ts b/modules/module-drizzle-storage/drizzle.sqlite.config.ts new file mode 100644 index 000000000..0b75bb676 --- /dev/null +++ b/modules/module-drizzle-storage/drizzle.sqlite.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'drizzle-kit'; + +export default defineConfig({ + dialect: 'sqlite', + schema: './src/drivers/sqlite/schema.ts', + out: './src/migrations/sqlite' +}); diff --git a/modules/module-drizzle-storage/package.json b/modules/module-drizzle-storage/package.json new file mode 100644 index 000000000..29165aa1a --- /dev/null +++ b/modules/module-drizzle-storage/package.json @@ -0,0 +1,51 @@ +{ + "name": "@powersync/service-module-drizzle-storage", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/index.d.ts", + "version": "0.1.0", + "main": "dist/index.js", + "license": "FSL-1.1-ALv2", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "rm -rf ./dist ./tsconfig.tsbuildinfo && tsc -b && cp -R src/migrations/sqlite dist/migrations/sqlite", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./dist && tsc -b --clean", + "drizzle:generate:sqlite": "drizzle-kit generate --config=drizzle.sqlite.config.ts", + "drizzle:check:sqlite": "drizzle-kit check --config=drizzle.sqlite.config.ts", + "test": "vitest" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "types": "./dist/types/types.d.ts", + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js" + } + }, + "dependencies": { + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-core": "workspace:*", + "@powersync/service-jsonbig": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "better-sqlite3": "^12.10.0", + "drizzle-orm": "^0.45.2", + "ts-codec": "^1.3.0", + "uuid": "catalog:" + }, + "devDependencies": { + "@powersync/service-core-tests": "workspace:*", + "@types/better-sqlite3": "^7.6.13", + "drizzle-kit": "^0.31.10", + "typescript": "catalog:" + } +} diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/SqliteDrizzleStorageFactory.ts b/modules/module-drizzle-storage/src/drivers/sqlite/SqliteDrizzleStorageFactory.ts new file mode 100644 index 000000000..23408da92 --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/SqliteDrizzleStorageFactory.ts @@ -0,0 +1,17 @@ +import { DrizzleBucketStorageFactory } from '../../storage/DrizzleBucketStorageFactory.js'; +import type { NormalizedDrizzleSqliteStorageConfig } from '../../types/types.js'; +import { createSqliteDrizzleRuntime, type SqliteDrizzleRuntime } from './sqlite-config.js'; +import { createSqliteDrizzleStorageDialect } from './sqlite-dialect.js'; + +export function createSqliteDrizzleStorageFactory(options: { + config: NormalizedDrizzleSqliteStorageConfig; + slotNamePrefix: string; + runtime?: SqliteDrizzleRuntime; +}): DrizzleBucketStorageFactory { + const runtime = options.runtime ?? createSqliteDrizzleRuntime(options.config); + return new DrizzleBucketStorageFactory({ + runtime, + dialect: createSqliteDrizzleStorageDialect(runtime), + slotNamePrefix: options.slotNamePrefix + }); +} diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts b/modules/module-drizzle-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts new file mode 100644 index 000000000..9c83284b9 --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts @@ -0,0 +1,76 @@ +import { locks } from '@powersync/lib-services-framework'; +import * as uuid from 'uuid'; +import type { SqliteDrizzleRuntime } from './sqlite-config.js'; + +const DEFAULT_LOCK_TIMEOUT = 60_000; + +export class SqliteMigrationLockManager extends locks.AbstractLockManager { + constructor(private readonly options: locks.LockManagerParams & { runtime: SqliteDrizzleRuntime }) { + super(options); + } + + private get timeout(): number { + return this.options.timeout ?? DEFAULT_LOCK_TIMEOUT; + } + + async init(): Promise { + this.options.runtime.client.exec(` + CREATE TABLE IF NOT EXISTS powersync_migration_locks ( + name TEXT PRIMARY KEY, + lock_id TEXT, + expires_at INTEGER NOT NULL + ) + `); + this.options.runtime.client + .prepare( + ` + INSERT OR IGNORE INTO powersync_migration_locks (name, lock_id, expires_at) + VALUES (?, NULL, 0) + ` + ) + .run(this.options.name); + } + + protected async acquireHandle(): Promise { + const lockId = uuid.v4(); + const now = Date.now(); + const result = this.options.runtime.client + .prepare( + ` + UPDATE powersync_migration_locks + SET lock_id = ?, expires_at = ? + WHERE name = ? AND (lock_id IS NULL OR expires_at <= ?) + ` + ) + .run(lockId, now + this.timeout, this.options.name, now); + if (result.changes != 1) { + return null; + } + return { + refresh: async () => { + const refreshed = this.options.runtime.client + .prepare( + ` + UPDATE powersync_migration_locks SET expires_at = ? WHERE name = ? AND lock_id = ? + ` + ) + .run(Date.now() + this.timeout, this.options.name, lockId); + if (refreshed.changes != 1) { + throw new Error('Lock not found, could not refresh'); + } + }, + release: async () => { + const released = this.options.runtime.client + .prepare( + ` + UPDATE powersync_migration_locks SET lock_id = NULL, expires_at = 0 WHERE name = ? AND lock_id = ? + ` + ) + .run(this.options.name, lockId); + if (released.changes != 1) { + throw new Error('Lock not found, could not release'); + } + } + }; + } +} diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/column-types.ts b/modules/module-drizzle-storage/src/drivers/sqlite/column-types.ts new file mode 100644 index 000000000..ceb54e8da --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/column-types.ts @@ -0,0 +1,55 @@ +import { customType } from 'drizzle-orm/sqlite-core'; + +/** + * SQLite INTEGER column whose application type is always bigint. + * + * The better-sqlite3 connections are configured with safe integers so the + * driver value reaches this mapper without first losing precision. + */ +export const sqliteBigInt = customType<{ data: bigint; driverData: bigint | number }>({ + dataType() { + return 'bigint'; + }, + toDriver(value) { + return value; + }, + fromDriver(value) { + return BigInt(value); + } +}); + +export const sqliteInteger = customType<{ data: number; driverData: bigint | number }>({ + dataType() { + return 'integer'; + }, + toDriver(value) { + return value; + }, + fromDriver(value) { + return Number(value); + } +}); + +export const sqliteBoolean = customType<{ data: boolean; driverData: bigint | number }>({ + dataType() { + return 'integer'; + }, + toDriver(value) { + return value ? 1 : 0; + }, + fromDriver(value) { + return value !== 0n && value !== 0; + } +}); + +export const sqliteTimestampMs = customType<{ data: Date; driverData: bigint | number }>({ + dataType() { + return 'integer'; + }, + toDriver(value) { + return value.getTime(); + }, + fromDriver(value) { + return new Date(Number(value)); + } +}); diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts b/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts new file mode 100644 index 000000000..6c6619592 --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts @@ -0,0 +1,137 @@ +import type { storage } from '@powersync/service-core'; +import { blob, index, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { sqliteBigInt, sqliteBoolean, sqliteInteger, sqliteTimestampMs } from './column-types.js'; + +export type CurrentBucket = { + bucket: string; + table: string; + id: string; +}; + +export const bucketData = sqliteTable( + 'bucket_data', + { + id: text('id').primaryKey(), + groupId: sqliteInteger('group_id').notNull(), + bucketName: text('bucket_name').notNull(), + opId: sqliteBigInt('op_id').notNull(), + op: text('op').notNull(), + sourceTable: text('source_table'), + sourceKey: blob('source_key', { mode: 'buffer' }), + tableName: text('table_name'), + rowId: text('row_id'), + checksum: sqliteBigInt('checksum').notNull(), + data: text('data'), + targetOp: sqliteBigInt('target_op') + }, + (table) => [ + index('bucket_data_bucket_op_index').on(table.groupId, table.bucketName, table.opId), + index('bucket_data_source_index').on(table.groupId, table.sourceTable, table.sourceKey) + ] +); + +export const bucketParameters = sqliteTable( + 'bucket_parameters', + { + id: sqliteBigInt('id').primaryKey(), + groupId: sqliteInteger('group_id').notNull(), + sourceTable: text('source_table').notNull(), + sourceKey: blob('source_key', { mode: 'buffer' }).notNull(), + lookup: blob('lookup', { mode: 'buffer' }).notNull(), + bucketParameters: text('bucket_parameters', { mode: 'json' }).$type().notNull() + }, + (table) => [ + index('bucket_parameters_lookup_index').on(table.groupId, table.lookup, table.id), + index('bucket_parameters_source_index').on(table.groupId, table.sourceTable, table.sourceKey) + ] +); + +export const currentData = sqliteTable( + 'current_data', + { + id: text('id').primaryKey(), + groupId: sqliteInteger('group_id').notNull(), + sourceTable: text('source_table').notNull(), + sourceKey: blob('source_key', { mode: 'buffer' }).notNull(), + buckets: text('buckets', { mode: 'json' }).$type().notNull(), + lookups: text('lookups', { mode: 'json' }).$type().notNull(), + data: blob('data', { mode: 'buffer' }).notNull(), + pendingDelete: sqliteBigInt('pending_delete') + }, + (table) => [ + index('current_data_source_index').on(table.groupId, table.sourceTable, table.sourceKey), + index('current_data_pending_delete_index').on(table.groupId, table.pendingDelete) + ] +); + +export const instance = sqliteTable('instance', { + id: text('id').primaryKey() +}); + +export const sourceTables = sqliteTable( + 'source_tables', + { + id: text('id').primaryKey(), + groupId: sqliteInteger('group_id').notNull(), + connectionId: sqliteInteger('connection_id').notNull(), + relationId: text('relation_id', { mode: 'json' }).$type(), + schemaName: text('schema_name').notNull(), + tableName: text('table_name').notNull(), + replicaIdColumns: text('replica_id_columns', { mode: 'json' }).$type(), + snapshotDone: sqliteBoolean('snapshot_done').notNull().default(true), + snapshotTotalEstimatedCount: sqliteBigInt('snapshot_total_estimated_count'), + snapshotReplicatedCount: sqliteBigInt('snapshot_replicated_count'), + snapshotLastKey: blob('snapshot_last_key', { mode: 'buffer' }) + }, + (table) => [index('source_table_lookup').on(table.groupId, table.tableName)] +); + +export const syncRules = sqliteTable('sync_rules', { + id: integer('id').primaryKey({ autoIncrement: true }), + state: text('state').$type().notNull(), + snapshotDone: sqliteBoolean('snapshot_done').notNull().default(false), + snapshotLsn: text('snapshot_lsn'), + lastCheckpoint: sqliteBigInt('last_checkpoint'), + lastCheckpointLsn: text('last_checkpoint_lsn'), + noCheckpointBefore: text('no_checkpoint_before'), + slotName: text('slot_name').notNull(), + lastCheckpointTs: sqliteTimestampMs('last_checkpoint_ts'), + lastKeepaliveTs: sqliteTimestampMs('last_keepalive_ts'), + lastFatalError: text('last_fatal_error'), + lastFatalErrorTs: sqliteTimestampMs('last_fatal_error_ts'), + keepaliveOp: sqliteBigInt('keepalive_op'), + storageVersion: sqliteInteger('storage_version'), + content: text('content').notNull(), + syncPlan: text('sync_plan', { mode: 'json' }).$type() +}); + +export const writeCheckpoints = sqliteTable( + 'write_checkpoints', + { + id: text('id').primaryKey(), + syncRulesId: sqliteInteger('sync_rules_id'), + userId: text('user_id').notNull(), + checkpoint: sqliteBigInt('checkpoint').notNull(), + heads: text('heads', { mode: 'json' }).$type>(), + createdAt: sqliteTimestampMs('created_at').notNull() + }, + (table) => [index('write_checkpoints_user_checkpoint_index').on(table.userId, table.syncRulesId, table.checkpoint)] +); + +export const sqliteSchema = { + bucketData, + bucketParameters, + currentData, + instance, + sourceTables, + syncRules, + writeCheckpoints +}; + +export type BucketDataRow = typeof bucketData.$inferSelect; +export type BucketParametersRow = typeof bucketParameters.$inferSelect; +export type CurrentDataRow = typeof currentData.$inferSelect; +export type InstanceRow = typeof instance.$inferSelect; +export type SourceTableRow = typeof sourceTables.$inferSelect; +export type SyncRulesRow = typeof syncRules.$inferSelect; +export type WriteCheckpointRow = typeof writeCheckpoints.$inferSelect; diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-config.ts b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-config.ts new file mode 100644 index 000000000..1fe59fb44 --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-config.ts @@ -0,0 +1,66 @@ +import Database from 'better-sqlite3'; +import type { ExtractTablesWithRelations } from 'drizzle-orm'; +import type { BetterSQLiteTransaction } from 'drizzle-orm/better-sqlite3'; +import { drizzle, type BetterSQLite3Database } from 'drizzle-orm/better-sqlite3'; +import type { NormalizedDrizzleSqliteStorageConfig } from '../../types/types.js'; +import { sqliteSchema } from './schema.js'; + +export type DrizzleStorageDatabase = BetterSQLite3Database & { + $client: Database.Database; +}; +export type DrizzleStorageTransaction = BetterSQLiteTransaction< + typeof sqliteSchema, + ExtractTablesWithRelations +>; + +export interface SqliteDrizzleRuntime { + readonly db: DrizzleStorageDatabase; + readonly client: Database.Database; + readonly readers: readonly DrizzleStorageDatabase[]; + read(): DrizzleStorageDatabase; + transaction(callback: (tx: DrizzleStorageTransaction) => T): T; + close(): void; +} + +export function createSqliteDrizzleRuntime(config: NormalizedDrizzleSqliteStorageConfig): SqliteDrizzleRuntime { + const fileBacked = config.filename != ':memory:'; + const client = openConnection(config.filename, fileBacked); + const db = drizzle(client, { schema: sqliteSchema }); + const readerCount = fileBacked ? Math.max(0, config.max_pool_size - 1) : 0; + const readerClients = Array.from({ length: readerCount }, () => openConnection(config.filename, true, true)); + const readers = readerClients.map((reader) => drizzle(reader, { schema: sqliteSchema })); + let nextReader = 0; + + return { + db, + client, + readers, + read() { + if (readers.length == 0) { + return db; + } + const reader = readers[nextReader % readers.length]; + nextReader++; + return reader; + }, + transaction(callback) { + return db.transaction(callback); + }, + close() { + for (const reader of readerClients) { + reader.close(); + } + client.close(); + } + }; +} + +function openConnection(filename: string, enableWal: boolean, readonly = false): Database.Database { + const client = new Database(filename, { readonly, fileMustExist: readonly }); + client.defaultSafeIntegers(true); + if (enableWal && !readonly) { + client.pragma('journal_mode = WAL'); + } + client.pragma('busy_timeout = 5000'); + return client; +} diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts new file mode 100644 index 000000000..8312b7b88 --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts @@ -0,0 +1,84 @@ +import { InProcessDrizzleCheckpointWatcher, type DrizzleStorageDialect } from '../../storage/DrizzleStorageDialect.js'; +import { DRIZZLE_SQLITE_STORAGE_TYPE } from '../../types/types.js'; +import { sqliteSchema } from './schema.js'; +import type { SqliteDrizzleRuntime } from './sqlite-config.js'; + +export function createSqliteDrizzleStorageDialect(runtime: SqliteDrizzleRuntime): DrizzleStorageDialect { + return { + type: DRIZZLE_SQLITE_STORAGE_TYPE, + db: runtime.db, + tables: sqliteSchema, + transaction: (callback) => runtime.transaction(callback), + async *streamBucketDataRows(options) { + if (options.dataBuckets.length == 0) { + return; + } + + await new Promise((resolve) => setImmediate(resolve)); + const db = options.db ?? runtime.read(); + const sortedBuckets = [...options.dataBuckets].sort((a, b) => a.bucket.localeCompare(b.bucket)); + let remaining = options.limit; + const statement = db.$client + .prepare( + ` + SELECT id, group_id, bucket_name, op_id, op, source_table, source_key, + table_name, row_id, checksum, data, target_op + FROM bucket_data + WHERE group_id = ? AND bucket_name = ? AND op_id > ? AND op_id <= ? + ORDER BY op_id ASC + LIMIT ? + ` + ) + .safeIntegers(true); + for (const request of sortedBuckets) { + if (remaining <= 0) { + return; + } + const rows = statement.iterate( + options.groupId, + request.bucket, + request.start, + options.checkpoint, + remaining + ) as Iterable; + for (const row of rows) { + yield mapBucketDataRow(row); + remaining--; + } + } + }, + createCheckpointWatcher: () => new InProcessDrizzleCheckpointWatcher() + }; +} + +interface RawBucketDataRow { + id: string; + group_id: bigint; + bucket_name: string; + op_id: bigint; + op: string; + source_table: string | null; + source_key: Buffer | null; + table_name: string | null; + row_id: string | null; + checksum: bigint; + data: string | null; + target_op: bigint | null; +} + +function mapBucketDataRow(row: RawBucketDataRow) { + return { + id: row.id, + groupId: Number(row.group_id), + bucketName: row.bucket_name, + opId: row.op_id, + op: row.op, + sourceTable: row.source_table, + sourceKey: row.source_key, + tableName: row.table_name, + rowId: row.row_id, + checksum: row.checksum, + data: row.data, + targetOp: row.target_op + }; +} diff --git a/modules/module-drizzle-storage/src/index.ts b/modules/module-drizzle-storage/src/index.ts new file mode 100644 index 000000000..110fd84d6 --- /dev/null +++ b/modules/module-drizzle-storage/src/index.ts @@ -0,0 +1,10 @@ +export * from './drivers/sqlite/schema.js'; +export * from './drivers/sqlite/sqlite-config.js'; +export * from './drivers/sqlite/sqlite-dialect.js'; +export * from './drivers/sqlite/SqliteDrizzleStorageFactory.js'; +export * from './migrations/DrizzleMigrationAgent.js'; +export * from './module/DrizzleStorageModule.js'; +export * from './storage/storage-index.js'; +export * as storage from './storage/storage-index.js'; +export * from './types/records.js'; +export * from './types/types.js'; diff --git a/modules/module-drizzle-storage/src/migrations/DrizzleMigrationAgent.ts b/modules/module-drizzle-storage/src/migrations/DrizzleMigrationAgent.ts new file mode 100644 index 000000000..5e77222ea --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/DrizzleMigrationAgent.ts @@ -0,0 +1,58 @@ +import * as framework from '@powersync/lib-services-framework'; +import { migrations } from '@powersync/service-core'; +import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import { SqliteMigrationLockManager } from '../drivers/sqlite/SqliteMigrationLockManager.js'; +import { createSqliteDrizzleRuntime, type SqliteDrizzleRuntime } from '../drivers/sqlite/sqlite-config.js'; +import { normalizeDrizzleSqliteStorageConfig, type DrizzleStorageConfigDecoded } from '../types/types.js'; +import { NoOpMigrationStore } from './NoOpMigrationStore.js'; + +export const SQLITE_DRIZZLE_MIGRATIONS_PATH = new URL('./sqlite', import.meta.url).pathname; + +export function runSqliteDrizzleMigrations(runtime: SqliteDrizzleRuntime): void { + migrate(runtime.db, { migrationsFolder: SQLITE_DRIZZLE_MIGRATIONS_PATH }); +} + +export class DrizzleMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent { + store: framework.MigrationStore = new NoOpMigrationStore(); + locks: framework.LockManager; + private readonly runtime: SqliteDrizzleRuntime; + + constructor(config: DrizzleStorageConfigDecoded) { + super(); + this.runtime = createSqliteDrizzleRuntime(normalizeDrizzleSqliteStorageConfig(config)); + this.locks = new SqliteMigrationLockManager({ + name: 'drizzle-migrations', + runtime: this.runtime + }); + } + + getInternalScriptsDir(): string { + return new URL('./scripts', import.meta.url).pathname; + } + + async run(params: framework.RunMigrationParams): Promise { + if (params.direction != framework.Direction.Up) { + throw new Error('Drizzle storage migrations only support the up direction'); + } + await this.locks.init?.(); + const lock = await this.locks.acquire({ + max_wait_ms: params.maxLockWaitMs ?? framework.DEFAULT_MAX_LOCK_WAIT_MS + }); + if (lock == null) { + throw new Error('Could not acquire Drizzle migration lock'); + } + try { + runSqliteDrizzleMigrations(this.runtime); + } finally { + await lock.release(); + } + } + + async loadInternalMigrations(): Promise[]> { + return []; + } + + async [Symbol.asyncDispose](): Promise { + this.runtime.close(); + } +} diff --git a/modules/module-drizzle-storage/src/migrations/NoOpMigrationStore.ts b/modules/module-drizzle-storage/src/migrations/NoOpMigrationStore.ts new file mode 100644 index 000000000..5e7ea01e0 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/NoOpMigrationStore.ts @@ -0,0 +1,9 @@ +import { migrations } from '@powersync/lib-services-framework'; + +export class NoOpMigrationStore implements migrations.MigrationStore { + async load(): Promise { + return undefined; + } + async save(_state: migrations.MigrationState): Promise {} + async clear(): Promise {} +} diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/0000_real_harpoon.sql b/modules/module-drizzle-storage/src/migrations/sqlite/0000_real_harpoon.sql new file mode 100644 index 000000000..a84d542a9 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/0000_real_harpoon.sql @@ -0,0 +1,89 @@ +CREATE TABLE `bucket_data` ( + `id` text PRIMARY KEY NOT NULL, + `group_id` integer NOT NULL, + `bucket_name` text NOT NULL, + `op_id` bigint NOT NULL, + `op` text NOT NULL, + `source_table` text, + `source_key` blob, + `table_name` text, + `row_id` text, + `checksum` bigint NOT NULL, + `data` text, + `target_op` bigint +); +--> statement-breakpoint +CREATE INDEX `bucket_data_bucket_op_index` ON `bucket_data` (`group_id`,`bucket_name`,`op_id`);--> statement-breakpoint +CREATE INDEX `bucket_data_source_index` ON `bucket_data` (`group_id`,`source_table`,`source_key`);--> statement-breakpoint +CREATE TABLE `bucket_parameters` ( + `id` bigint PRIMARY KEY NOT NULL, + `group_id` integer NOT NULL, + `source_table` text NOT NULL, + `source_key` blob NOT NULL, + `lookup` blob NOT NULL, + `bucket_parameters` text NOT NULL +); +--> statement-breakpoint +CREATE INDEX `bucket_parameters_lookup_index` ON `bucket_parameters` (`group_id`,`lookup`,`id`);--> statement-breakpoint +CREATE INDEX `bucket_parameters_source_index` ON `bucket_parameters` (`group_id`,`source_table`,`source_key`);--> statement-breakpoint +CREATE TABLE `current_data` ( + `id` text PRIMARY KEY NOT NULL, + `group_id` integer NOT NULL, + `source_table` text NOT NULL, + `source_key` blob NOT NULL, + `buckets` text NOT NULL, + `lookups` text NOT NULL, + `data` blob NOT NULL, + `pending_delete` bigint +); +--> statement-breakpoint +CREATE INDEX `current_data_source_index` ON `current_data` (`group_id`,`source_table`,`source_key`);--> statement-breakpoint +CREATE INDEX `current_data_pending_delete_index` ON `current_data` (`group_id`,`pending_delete`);--> statement-breakpoint +CREATE TABLE `instance` ( + `id` text PRIMARY KEY NOT NULL +); +--> statement-breakpoint +CREATE TABLE `source_tables` ( + `id` text PRIMARY KEY NOT NULL, + `group_id` integer NOT NULL, + `connection_id` integer NOT NULL, + `relation_id` text, + `schema_name` text NOT NULL, + `table_name` text NOT NULL, + `replica_id_columns` text, + `snapshot_done` integer DEFAULT true NOT NULL, + `snapshot_total_estimated_count` bigint, + `snapshot_replicated_count` bigint, + `snapshot_last_key` blob +); +--> statement-breakpoint +CREATE INDEX `source_table_lookup` ON `source_tables` (`group_id`,`table_name`);--> statement-breakpoint +CREATE TABLE `sync_rules` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `state` text NOT NULL, + `snapshot_done` integer DEFAULT false NOT NULL, + `snapshot_lsn` text, + `last_checkpoint` bigint, + `last_checkpoint_lsn` text, + `no_checkpoint_before` text, + `slot_name` text NOT NULL, + `last_checkpoint_ts` integer, + `last_keepalive_ts` integer, + `last_fatal_error` text, + `last_fatal_error_ts` integer, + `keepalive_op` bigint, + `storage_version` integer, + `content` text NOT NULL, + `sync_plan` text +); +--> statement-breakpoint +CREATE TABLE `write_checkpoints` ( + `id` text PRIMARY KEY NOT NULL, + `sync_rules_id` integer, + `user_id` text NOT NULL, + `checkpoint` bigint NOT NULL, + `heads` text, + `created_at` integer NOT NULL +); +--> statement-breakpoint +CREATE INDEX `write_checkpoints_user_checkpoint_index` ON `write_checkpoints` (`user_id`,`sync_rules_id`,`checkpoint`); \ No newline at end of file diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/meta/0000_snapshot.json b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0000_snapshot.json new file mode 100644 index 000000000..3122e1056 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0000_snapshot.json @@ -0,0 +1,555 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "d595e266-c1de-43fe-9e46-2deb2b424122", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "bucket_data": { + "name": "bucket_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_name": { + "name": "bucket_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op_id": { + "name": "op_id", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op": { + "name": "op", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checksum": { + "name": "checksum", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_op": { + "name": "target_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bucket_data_bucket_op_index": { + "name": "bucket_data_bucket_op_index", + "columns": ["group_id", "bucket_name", "op_id"], + "isUnique": false + }, + "bucket_data_source_index": { + "name": "bucket_data_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bucket_parameters": { + "name": "bucket_parameters", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookup": { + "name": "lookup", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_parameters": { + "name": "bucket_parameters", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bucket_parameters_lookup_index": { + "name": "bucket_parameters_lookup_index", + "columns": ["group_id", "lookup", "id"], + "isUnique": false + }, + "bucket_parameters_source_index": { + "name": "bucket_parameters_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "current_data": { + "name": "current_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "buckets": { + "name": "buckets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookups": { + "name": "lookups", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pending_delete": { + "name": "pending_delete", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "current_data_source_index": { + "name": "current_data_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + }, + "current_data_pending_delete_index": { + "name": "current_data_pending_delete_index", + "columns": ["group_id", "pending_delete"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance": { + "name": "instance", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "source_tables": { + "name": "source_tables", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connection_id": { + "name": "connection_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relation_id": { + "name": "relation_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schema_name": { + "name": "schema_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "replica_id_columns": { + "name": "replica_id_columns", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "snapshot_total_estimated_count": { + "name": "snapshot_total_estimated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_replicated_count": { + "name": "snapshot_replicated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_last_key": { + "name": "snapshot_last_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "source_table_lookup": { + "name": "source_table_lookup", + "columns": ["group_id", "table_name"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sync_rules": { + "name": "sync_rules", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "snapshot_lsn": { + "name": "snapshot_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint": { + "name": "last_checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint_lsn": { + "name": "last_checkpoint_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "no_checkpoint_before": { + "name": "no_checkpoint_before", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "slot_name": { + "name": "slot_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_checkpoint_ts": { + "name": "last_checkpoint_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_keepalive_ts": { + "name": "last_keepalive_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error": { + "name": "last_fatal_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error_ts": { + "name": "last_fatal_error_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "keepalive_op": { + "name": "keepalive_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storage_version": { + "name": "storage_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sync_plan": { + "name": "sync_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "write_checkpoints": { + "name": "write_checkpoints", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "sync_rules_id": { + "name": "sync_rules_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checkpoint": { + "name": "checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "heads": { + "name": "heads", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "write_checkpoints_user_checkpoint_index": { + "name": "write_checkpoints_user_checkpoint_index", + "columns": ["user_id", "sync_rules_id", "checkpoint"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json b/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json new file mode 100644 index 000000000..35ed86f69 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1783955536218, + "tag": "0000_real_harpoon", + "breakpoints": true + } + ] +} diff --git a/modules/module-drizzle-storage/src/module/DrizzleStorageModule.ts b/modules/module-drizzle-storage/src/module/DrizzleStorageModule.ts new file mode 100644 index 000000000..46a943e11 --- /dev/null +++ b/modules/module-drizzle-storage/src/module/DrizzleStorageModule.ts @@ -0,0 +1,21 @@ +import { modules, system } from '@powersync/service-core'; +import { DrizzleMigrationAgent } from '../migrations/DrizzleMigrationAgent.js'; +import { DrizzleStorageProvider } from '../storage/DrizzleStorageProvider.js'; +import { DrizzleStorageConfig, isDrizzleStorageConfig } from '../types/types.js'; + +export class DrizzleStorageModule extends modules.AbstractModule { + constructor() { + super({ name: 'Drizzle Bucket Storage' }); + } + + async initialize(context: system.ServiceContextContainer): Promise { + context.storageEngine.registerProvider(new DrizzleStorageProvider()); + if (isDrizzleStorageConfig(context.configuration.storage)) { + context.migrations.registerMigrationAgent( + new DrizzleMigrationAgent(DrizzleStorageConfig.decode(context.configuration.storage)) + ); + } + } + + async teardown(): Promise {} +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleBucketStorageFactory.ts b/modules/module-drizzle-storage/src/storage/DrizzleBucketStorageFactory.ts new file mode 100644 index 000000000..2fd293595 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleBucketStorageFactory.ts @@ -0,0 +1,239 @@ +import { DO_NOT_LOG } from '@powersync/lib-services-framework'; +import { GetIntanceOptions, storage } from '@powersync/service-core'; +import crypto from 'crypto'; +import { desc, eq, inArray } from 'drizzle-orm'; +import * as uuid from 'uuid'; +import type { BucketDataRow, BucketParametersRow, CurrentDataRow, SyncRulesRow } from '../drivers/sqlite/schema.js'; +import type { SqliteDrizzleRuntime } from '../drivers/sqlite/sqlite-config.js'; +import { DrizzlePersistedReplicationStream } from './DrizzlePersistedReplicationStream.js'; +import type { DrizzleCheckpointWatcher, DrizzleStorageDialect } from './DrizzleStorageDialect.js'; +import { DrizzleSyncRulesStorage } from './DrizzleSyncRulesStorage.js'; + +export interface DrizzleBucketStorageFactoryOptions { + runtime: SqliteDrizzleRuntime; + dialect: DrizzleStorageDialect; + slotNamePrefix: string; +} + +export class DrizzleBucketStorageFactory extends storage.BucketStorageFactory { + [DO_NOT_LOG] = true; + + readonly runtime: SqliteDrizzleRuntime; + readonly dialect: DrizzleStorageDialect; + readonly slotNamePrefix: string; + readonly checkpointWatcher: DrizzleCheckpointWatcher; + + constructor(options: DrizzleBucketStorageFactoryOptions) { + super(); + this.runtime = options.runtime; + this.dialect = options.dialect; + this.slotNamePrefix = options.slotNamePrefix; + this.checkpointWatcher = options.dialect.createCheckpointWatcher(); + } + + async [Symbol.asyncDispose](): Promise { + this.runtime.close(); + } + + getInstance( + replicationStream: storage.PersistedReplicationStream, + options?: GetIntanceOptions + ): storage.SyncRulesBucketStorage { + const syncRuleStorage = new DrizzleSyncRulesStorage({ + factory: this, + dialect: this.dialect, + replicationStream + }); + + if (!options?.skipLifecycleHooks) { + this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage)); + } + + syncRuleStorage.registerListener({ + batchStarted: (batch) => { + batch.registerListener({ + replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) + }); + } + }); + + return syncRuleStorage; + } + + async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { + const storageVersion = + options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; + if (storage.STORAGE_VERSION_CONFIG[storageVersion] == null) { + throw new Error(`Unsupported storage version ${storageVersion}`); + } + + const tables = this.dialect.tables; + const row = this.dialect.transaction((tx) => { + tx.update(tables.syncRules) + .set({ state: storage.SyncRuleState.STOP }) + .where(eq(tables.syncRules.state, storage.SyncRuleState.PROCESSING)) + .run(); + + return tx + .insert(tables.syncRules) + .values({ + state: storage.SyncRuleState.PROCESSING, + snapshotDone: false, + snapshotLsn: null, + lastCheckpoint: null, + lastCheckpointLsn: null, + noCheckpointBefore: null, + slotName: this.generateReplicationStreamName(), + lastCheckpointTs: null, + lastKeepaliveTs: null, + lastFatalError: null, + lastFatalErrorTs: null, + keepaliveOp: null, + storageVersion, + content: options.config.yaml, + syncPlan: options.config.plan + }) + .returning() + .get(); + }); + + return new DrizzlePersistedReplicationStream(this.dialect, row); + } + + async restartReplication(replicationStreamId: number): Promise { + const active = await this.getActiveSyncConfig(); + const deploying = await this.getDeployingSyncConfig(); + const stream = + deploying?.replicationStream.replicationStreamId == replicationStreamId + ? deploying + : active?.replicationStream.replicationStreamId == replicationStreamId + ? active + : null; + if (stream != null) { + await this.updateSyncRules(stream.content.asUpdateOptions()); + } + } + + async getActiveSyncConfig(): Promise { + return this.getSyncConfigForStates([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED]); + } + + async getDeployingSyncConfig(): Promise { + return this.getSyncConfigForStates([storage.SyncRuleState.PROCESSING]); + } + + async getReplicatingReplicationStreams(): Promise { + return this.findSyncRulesRows([storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]).map( + (row) => new DrizzlePersistedReplicationStream(this.dialect, row) + ); + } + + async getStoppedReplicationStreams(): Promise { + return this.findSyncRulesRows([storage.SyncRuleState.STOP]).map( + (row) => new DrizzlePersistedReplicationStream(this.dialect, row) + ); + } + + async getStorageMetrics(): Promise { + const { db, tables } = this.dialect; + const operations = db.select().from(tables.bucketData).all(); + const parameters = db.select().from(tables.bucketParameters).all(); + const currentData = db.select().from(tables.currentData).all(); + return { + operations_size_bytes: operations.reduce((total, row) => total + estimateBucketDataSize(row), 0), + parameters_size_bytes: parameters.reduce((total, row) => total + estimateBucketParametersSize(row), 0), + replication_size_bytes: currentData.reduce((total, row) => total + estimateCurrentDataSize(row), 0) + }; + } + + async getPowerSyncInstanceId(): Promise { + const { db, tables } = this.dialect; + const existing = db.select().from(tables.instance).limit(1).get(); + if (existing != null) { + return existing.id; + } + const id = uuid.v4(); + db.insert(tables.instance).values({ id }).onConflictDoNothing().run(); + return db.select().from(tables.instance).limit(1).get()!.id; + } + + async getSystemIdentifier(): Promise { + return { + id: `${this.dialect.type}:${await this.getPowerSyncInstanceId()}`, + type: this.dialect.type + }; + } + + private async getSyncConfigForStates(states: storage.SyncRuleState[]): Promise { + const [row] = this.findSyncRulesRows(states); + if (row == null) { + return null; + } + const replicationStream = new DrizzlePersistedReplicationStream(this.dialect, row); + return { + content: replicationStream.syncConfigContent[0], + replicationStream, + storage: this.getInstance(replicationStream, { skipLifecycleHooks: true }) + }; + } + + private findSyncRulesRows(states: storage.SyncRuleState[]): SyncRulesRow[] { + const { db, tables } = this.dialect; + if (states.length == 0) { + return []; + } + return db + .select() + .from(tables.syncRules) + .where(inArray(tables.syncRules.state, states)) + .orderBy(desc(tables.syncRules.id)) + .all(); + } + + private generateReplicationStreamName(): string { + return `${this.slotNamePrefix}${Date.now()}_${crypto.randomBytes(2).toString('hex')}`; + } +} + +function estimateString(value: string | null | undefined): number { + return value == null ? 0 : value.length; +} +function estimateBuffer(value: Buffer | Uint8Array | null | undefined): number { + return value == null ? 0 : value.byteLength; +} +function estimateJson(value: unknown): number { + return typeof value == 'string' ? value.length : JSON.stringify(value ?? null).length; +} +function estimateBucketDataSize(row: BucketDataRow): number { + return ( + 80 + + estimateString(row.id) + + estimateString(row.bucketName) + + estimateString(row.op) + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateString(row.tableName) + + estimateString(row.rowId) + + estimateString(row.data) + ); +} +function estimateBucketParametersSize(row: BucketParametersRow): number { + return ( + 80 + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateBuffer(row.lookup) + + estimateJson(row.bucketParameters) + ); +} +function estimateCurrentDataSize(row: CurrentDataRow): number { + return ( + 80 + + estimateString(row.id) + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateJson(row.buckets) + + estimateJson(row.lookups) + + estimateBuffer(row.data) + ); +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts b/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts new file mode 100644 index 000000000..ab396bbeb --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts @@ -0,0 +1,75 @@ +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { eq } from 'drizzle-orm'; +import type { SyncRulesRow } from '../drivers/sqlite/schema.js'; +import type { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +export class DrizzlePersistedSyncConfigContent extends storage.PersistedSyncConfigContent { + constructor( + private readonly dialect: DrizzleStorageDialect, + row: SyncRulesRow + ) { + super({ + replicationStreamId: row.id, + sync_rules_content: row.content, + compiled_plan: row.syncPlan, + replicationStreamName: row.slotName, + storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION, + syncConfigId: String(row.id) + }); + } + + async getSyncConfigStatus(): Promise { + const { db, tables } = this.dialect; + const row = db.select().from(tables.syncRules).where(eq(tables.syncRules.id, this.replicationStreamId)).get(); + return row == null ? null : syncConfigStatusFromRow(row); + } +} + +export class DrizzlePersistedReplicationStream extends storage.PersistedReplicationStream { + current_lock: storage.ReplicationLock | null = null; + readonly syncConfigContent: readonly DrizzlePersistedSyncConfigContent[]; + + constructor( + private readonly dialect: DrizzleStorageDialect, + private readonly row: SyncRulesRow + ) { + super({ + replicationStreamId: row.id, + replicationStreamName: row.slotName, + state: row.state, + storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION + }); + this.syncConfigContent = [new DrizzlePersistedSyncConfigContent(this.dialect, this.row)]; + } + + parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet { + return this.syncConfigContent[0].parsed(options); + } + + async lock(): Promise { + if (this.current_lock != null) { + throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by this process.`); + } + return (this.current_lock = { + sync_rules_id: this.replicationStreamId, + release: async () => { + this.current_lock = null; + } + }); + } +} + +export function syncConfigStatusFromRow(row: SyncRulesRow): storage.PersistedSyncConfigStatus { + return { + id: String(row.id), + replicationStreamId: row.id, + state: row.state, + snapshot_done: row.snapshotDone, + last_checkpoint_lsn: row.lastCheckpointLsn, + last_fatal_error: row.lastFatalError, + last_fatal_error_ts: row.lastFatalErrorTs, + last_keepalive_ts: row.lastKeepaliveTs, + last_checkpoint_ts: row.lastCheckpointTs + }; +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleReportStorage.ts b/modules/module-drizzle-storage/src/storage/DrizzleReportStorage.ts new file mode 100644 index 000000000..1cb4de175 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleReportStorage.ts @@ -0,0 +1,34 @@ +import { storage } from '@powersync/service-core'; +import { event_types } from '@powersync/service-types'; + +export class DrizzleReportStorage implements storage.ReportStorage { + async [Symbol.asyncDispose](): Promise { + // Report storage is intentionally a no-op in the initial Drizzle storage slice. + } + + async reportClientConnection(_data: event_types.ClientConnectionBucketData): Promise {} + + async reportClientDisconnection(_data: event_types.ClientDisconnectionEventData): Promise {} + + async getConnectedClients(): Promise { + return { users: [], sdks: [] } as unknown as event_types.ClientConnectionReportResponse; + } + + async getClientConnectionReports( + _data: event_types.ClientConnectionReportRequest + ): Promise { + return { users: [], sdks: [] } as unknown as event_types.ClientConnectionReportResponse; + } + + async getGeneralClientConnectionAnalytics( + _data: event_types.ClientConnectionAnalyticsRequest + ): Promise> { + return { + items: [], + count: 0, + more: false + } as unknown as event_types.PaginatedResponse; + } + + async deleteOldConnectionData(_data: event_types.DeleteOldConnectionData): Promise {} +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts b/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts new file mode 100644 index 000000000..1a1346391 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts @@ -0,0 +1,55 @@ +import type { BucketDataRequest } from '@powersync/service-core'; +import type { BucketDataRow, sqliteSchema } from '../drivers/sqlite/schema.js'; +import type { DrizzleStorageDatabase, DrizzleStorageTransaction } from '../drivers/sqlite/sqlite-config.js'; + +export interface DrizzleStorageDialect { + readonly type: string; + readonly db: DrizzleStorageDatabase; + readonly tables: typeof sqliteSchema; + transaction(callback: (tx: DrizzleStorageTransaction) => T): T; + streamBucketDataRows(options: DrizzleBucketDataStreamOptions): AsyncIterable; + createCheckpointWatcher(): DrizzleCheckpointWatcher; +} + +export interface DrizzleBucketDataStreamOptions { + readonly db?: DrizzleStorageDatabase; + readonly groupId: number; + readonly checkpoint: bigint; + readonly dataBuckets: BucketDataRequest[]; + readonly limit: number; +} + +export interface DrizzleCheckpointWatcher { + notify(): void; + watch(signal: AbortSignal): AsyncIterable; +} + +export class InProcessDrizzleCheckpointWatcher implements DrizzleCheckpointWatcher { + private readonly listeners = new Set<() => void>(); + + notify(): void { + for (const listener of this.listeners) { + listener(); + } + } + + async *watch(signal: AbortSignal): AsyncIterable { + while (!signal.aborted) { + yield await new Promise((resolve) => { + const listener = () => { + this.listeners.delete(listener); + resolve(); + }; + this.listeners.add(listener); + signal.addEventListener( + 'abort', + () => { + this.listeners.delete(listener); + resolve(); + }, + { once: true } + ); + }); + } + } +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleStorageProvider.ts b/modules/module-drizzle-storage/src/storage/DrizzleStorageProvider.ts new file mode 100644 index 000000000..0a4e8abc7 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleStorageProvider.ts @@ -0,0 +1,67 @@ +import { logger } from '@powersync/lib-services-framework'; +import { storage, system } from '@powersync/service-core'; +import { createSqliteDrizzleStorageFactory } from '../drivers/sqlite/SqliteDrizzleStorageFactory.js'; +import { + DRIZZLE_SQLITE_STORAGE_TYPE, + DrizzleSqliteStorageConfig, + isDrizzleStorageConfig, + normalizeDrizzleSqliteStorageConfig +} from '../types/types.js'; +import { DrizzleReportStorage } from './DrizzleReportStorage.js'; + +export class DrizzleStorageProvider implements storage.StorageProvider { + get type(): typeof DRIZZLE_SQLITE_STORAGE_TYPE { + return DRIZZLE_SQLITE_STORAGE_TYPE; + } + + async getStorage(options: storage.GetStorageOptions): Promise { + const storageConfig = options.resolvedConfig.storage; + if (!isDrizzleStorageConfig(storageConfig)) { + throw new Error(`Cannot create Drizzle storage with provided config ${storageConfig.type}`); + } + assertSqliteServiceMode(options.serviceMode); + const normalizedConfig = normalizeDrizzleSqliteStorageConfig(DrizzleSqliteStorageConfig.decode(storageConfig)); + const factory = createSqliteDrizzleStorageFactory({ + config: normalizedConfig, + slotNamePrefix: options.resolvedConfig.slot_name_prefix + }); + + return { + reportStorage: new DrizzleReportStorage(), + storage: factory, + shutDown: async () => factory[Symbol.asyncDispose](), + tearDown: async () => { + logger.info(`Tearing down Drizzle SQLite storage: ${normalizedConfig.filename}...`); + for (const table of [ + 'write_checkpoints', + 'bucket_parameters', + 'current_data', + 'bucket_data', + 'source_tables', + 'sync_rules', + 'instance' + ]) { + factory.runtime.client.exec(`DROP TABLE IF EXISTS \`${table}\``); + } + await factory[Symbol.asyncDispose](); + return true; + } + }; + } +} + +const SQLITE_ALLOWED_COMMAND_MODES = new Set([ + system.ServiceContextMode.COMPACT, + system.ServiceContextMode.TEARDOWN, + system.ServiceContextMode.TEST_CONNECTION +]); + +function assertSqliteServiceMode(serviceMode: string): void { + if (serviceMode == system.ServiceContextMode.UNIFIED || SQLITE_ALLOWED_COMMAND_MODES.has(serviceMode)) { + return; + } + throw new Error( + `Drizzle SQLite storage only supports the unified service runner. ` + + `SQLite checkpoint notifications are process-local, so split runners cannot safely share this storage.` + ); +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts b/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts new file mode 100644 index 000000000..d79e3070e --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts @@ -0,0 +1,573 @@ +import { BaseObserver, DO_NOT_LOG, errors, Logger } from '@powersync/lib-services-framework'; +import { + BucketChecksumRequest, + BucketDataBatchOptions, + BucketDataRequest, + CHECKPOINT_INVALIDATE_ALL, + CheckpointChanges, + GetCheckpointChangesOptions, + PopulateChecksumCacheOptions, + PopulateChecksumCacheResults, + ReplicationCheckpoint, + storage, + StorageCheckpointUpdate, + SyncBucketDataChunk, + utils, + WatchWriteCheckpointOptions +} from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import { and, asc, desc, eq, isNull, lte } from 'drizzle-orm'; +import * as uuid from 'uuid'; +import type { BucketDataRow } from '../drivers/sqlite/schema.js'; +import { DrizzleBucketBatch } from './DrizzleBucketBatch.js'; +import { DrizzleBucketStorageFactory } from './DrizzleBucketStorageFactory.js'; +import { DrizzleCompactor } from './DrizzleCompactor.js'; +import { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +export interface DrizzleSyncRulesStorageOptions { + factory: DrizzleBucketStorageFactory; + dialect: DrizzleStorageDialect; + replicationStream: storage.PersistedReplicationStream; +} + +export class DrizzleSyncRulesStorage + extends BaseObserver + implements storage.SyncRulesBucketStorage +{ + [DO_NOT_LOG] = true; + + readonly replicationStreamId: number; + readonly replicationStreamName: string; + readonly storageConfig: storage.StorageVersionConfig; + readonly factory: DrizzleBucketStorageFactory; + readonly logger: Logger; + + private parsedSyncRulesCache: + | { parsed: sync_rules.HydratedSyncConfig; options: storage.ParseSyncConfigOptions } + | undefined; + + private writeCheckpointModeValue = storage.WriteCheckpointMode.MANAGED; + + constructor(private readonly options: DrizzleSyncRulesStorageOptions) { + super(); + this.replicationStreamId = options.replicationStream.replicationStreamId; + this.replicationStreamName = options.replicationStream.replicationStreamName; + this.storageConfig = options.replicationStream.getStorageConfig(); + this.factory = options.factory; + this.logger = options.replicationStream.logger; + } + + get writeCheckpointMode(): storage.WriteCheckpointMode { + return this.writeCheckpointModeValue; + } + + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { + this.writeCheckpointModeValue = mode; + } + + async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { + if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) { + throw new errors.ValidationError( + `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` + ); + } + + const table = this.options.dialect.tables.writeCheckpoints; + const createdCheckpoint = this.options.dialect.transaction((tx) => { + const latest = tx + .select() + .from(table) + .where(and(eq(table.userId, checkpoint.user_id), isNull(table.syncRulesId))) + .orderBy(desc(table.checkpoint)) + .limit(1) + .get(); + const value = (latest?.checkpoint ?? 0n) + 1n; + tx.insert(table) + .values({ + id: uuid.v4(), + syncRulesId: null, + userId: checkpoint.user_id, + checkpoint: value, + heads: checkpoint.heads, + createdAt: new Date() + }) + .run(); + return value; + }); + + this.factory.checkpointWatcher.notify(); + return createdCheckpoint; + } + + async lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { + switch (this.writeCheckpointMode) { + case storage.WriteCheckpointMode.CUSTOM: + return this.lastCustomWriteCheckpoint({ + user_id: filters.user_id, + sync_rules_id: this.replicationStreamId + }); + case storage.WriteCheckpointMode.MANAGED: + if (!('heads' in filters)) { + throw new errors.ValidationError(`Replication HEAD is required for managed Write Checkpoint filtering`); + } + return this.lastManagedWriteCheckpoint(filters); + } + } + + async createWriter(options: storage.CreateWriterOptions): Promise { + const { db, tables } = this.options.dialect; + const syncRules = db.select().from(tables.syncRules).where(eq(tables.syncRules.id, this.replicationStreamId)).get(); + + const checkpointLsn = syncRules?.lastCheckpointLsn ?? null; + const writer = new DrizzleBucketBatch({ + factory: this.factory, + dialect: this.options.dialect, + logger: options.logger ?? this.logger, + syncRules: this.getParsedSyncRules(options), + replicationStreamId: this.replicationStreamId, + replicationStreamName: this.replicationStreamName, + lastCheckpointLsn: checkpointLsn, + keepaliveOp: syncRules?.keepaliveOp ?? null, + resumeFromLsn: utils.maxLsn(syncRules?.snapshotLsn, checkpointLsn), + storeCurrentData: options.storeCurrentData, + skipExistingRows: options.skipExistingRows ?? false, + markRecordUnavailable: options.markRecordUnavailable, + hooks: options.hooks + }); + this.iterateListeners((cb) => cb.batchStarted?.(writer)); + return writer; + } + + async startBatch( + options: storage.CreateWriterOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + await using writer = await this.createWriter(options); + await callback(writer); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + } + + getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig { + const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; + if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { + this.parsedSyncRulesCache = { + parsed: this.options.replicationStream.parsed(options).hydratedSyncConfig(), + options + }; + } + + return this.parsedSyncRulesCache!.parsed; + } + + async terminate(options?: storage.TerminateOptions): Promise { + if (!options || options.clearStorage) { + await this.clear(options); + } + + const { db, tables } = this.options.dialect; + db.update(tables.syncRules) + .set({ + state: storage.SyncRuleState.TERMINATED, + snapshotDone: false + }) + .where(eq(tables.syncRules.id, this.replicationStreamId)) + .run(); + this.factory.checkpointWatcher.notify(); + } + + async getStatus(): Promise { + const { db, tables } = this.options.dialect; + const row = db.select().from(tables.syncRules).where(eq(tables.syncRules.id, this.replicationStreamId)).get(); + + return { + checkpoint_lsn: row?.lastCheckpointLsn ?? null, + active: row?.state == storage.SyncRuleState.ACTIVE, + snapshot_done: row?.snapshotDone ?? false, + snapshot_lsn: row?.snapshotLsn ?? null, + keepalive_op: row?.keepaliveOp ?? null + }; + } + + async clear(_options?: storage.ClearStorageOptions): Promise { + const tables = this.options.dialect.tables; + this.options.dialect.transaction((tx) => { + tx.update(tables.syncRules) + .set({ + snapshotDone: false, + lastCheckpointLsn: null, + lastCheckpoint: null, + noCheckpointBefore: null + }) + .where(eq(tables.syncRules.id, this.replicationStreamId)) + .run(); + tx.delete(tables.bucketData).where(eq(tables.bucketData.groupId, this.replicationStreamId)).run(); + tx.delete(tables.bucketParameters).where(eq(tables.bucketParameters.groupId, this.replicationStreamId)).run(); + tx.delete(tables.currentData).where(eq(tables.currentData.groupId, this.replicationStreamId)).run(); + tx.delete(tables.sourceTables).where(eq(tables.sourceTables.groupId, this.replicationStreamId)).run(); + }); + + this.factory.checkpointWatcher.notify(); + } + + async reportError(e: any): Promise { + const { db, tables } = this.options.dialect; + db.update(tables.syncRules) + .set({ + lastFatalError: String(e.message ?? 'Replication failure'), + lastFatalErrorTs: new Date() + }) + .where(eq(tables.syncRules.id, this.replicationStreamId)) + .run(); + } + + async compact(options?: storage.CompactOptions): Promise { + let maxOpId = options?.maxOpId; + if (maxOpId == null) { + const checkpoint = await this.getCheckpoint(); + maxOpId = checkpoint.checkpoint; + } + + const compactor = new DrizzleCompactor(this.options.dialect, this.replicationStreamId, { + ...options, + maxOpId, + logger: this.logger + }); + await compactor.compact(); + + if (options?.compactParameterData) { + await compactor.compactParameterData(options); + } + } + + async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise { + return { buckets: 0 }; + } + + async getCheckpoint(): Promise { + const { db, tables } = this.options.dialect; + const row = db.select().from(tables.syncRules).where(eq(tables.syncRules.id, this.replicationStreamId)).get(); + + return { + checkpoint: row?.lastCheckpoint ?? 0n, + lsn: row?.lastCheckpointLsn ?? null, + getParameterSets: (lookups, limit) => this.getParameterSets(row?.lastCheckpoint ?? 0n, lookups, limit) + }; + } + + async getCheckpointChanges(_options: GetCheckpointChangesOptions): Promise { + return CHECKPOINT_INVALIDATE_ALL; + } + + async *watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable { + let lastCheckpoint: bigint | null = null; + let lastCheckpointLsn: string | null = null; + let lastWriteCheckpoint: bigint | null = null; + const { signal, user_id } = options; + + const watcher = this.factory.checkpointWatcher.watch(signal)[Symbol.asyncIterator](); + let nextNotification: Promise> | null = null; + let readImmediately = true; + + try { + while (!signal.aborted) { + if (!readImmediately) { + nextNotification ??= watcher.next(); + const result = await nextNotification; + nextNotification = null; + if (result.done) { + return; + } + } + readImmediately = false; + + if (signal.aborted) { + return; + } + + const base = await this.getCheckpoint(); + const currentWriteCheckpoint = await this.lastWriteCheckpoint({ + user_id, + heads: base.lsn == null ? {} : { '1': base.lsn } + }); + + if ( + currentWriteCheckpoint == lastWriteCheckpoint && + base.checkpoint == lastCheckpoint && + base.lsn == lastCheckpointLsn + ) { + continue; + } + + lastWriteCheckpoint = currentWriteCheckpoint; + lastCheckpoint = base.checkpoint; + lastCheckpointLsn = base.lsn; + nextNotification = watcher.next(); + + yield { + base, + writeCheckpoint: currentWriteCheckpoint, + update: CHECKPOINT_INVALIDATE_ALL + }; + } + } finally { + await watcher.return?.(); + } + } + + async *getBucketDataBatch( + checkpoint: bigint, + dataBuckets: BucketDataRequest[], + options?: BucketDataBatchOptions + ): AsyncIterable { + if (dataBuckets.length == 0) { + return; + } + + const batchRowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + const startOpByBucket = new Map(dataBuckets.map((request) => [request.bucket, request.start])); + const rows = this.options.dialect.streamBucketDataRows({ + groupId: this.replicationStreamId, + checkpoint, + dataBuckets, + limit: batchRowLimit + }); + + let chunkSizeBytes = 0; + let currentChunk: utils.SyncBucketData | null = null; + let targetOp: bigint | null = null; + let batchRowCount = 0; + + for await (const row of rows) { + const rowSizeBytes = row.data?.length ?? 0; + const sizeExceeded = + chunkSizeBytes >= chunkSizeLimitBytes || + ((currentChunk?.data.length ?? 0) > 0 && chunkSizeBytes + rowSizeBytes > chunkSizeLimitBytes) || + (currentChunk?.data.length ?? 0) >= batchRowLimit; + + if (currentChunk == null || currentChunk.bucket != row.bucketName || sizeExceeded) { + let start: string | undefined = undefined; + if (currentChunk != null) { + if (currentChunk.bucket == row.bucketName) { + currentChunk.has_more = true; + start = currentChunk.next_after; + } + + const yieldChunk = currentChunk; + currentChunk = null; + chunkSizeBytes = 0; + yield { chunkData: yieldChunk, targetOp }; + targetOp = null; + if (batchRowCount >= batchRowLimit) { + break; + } + } + + if (start == null) { + const startOpId = startOpByBucket.get(row.bucketName); + if (startOpId == null) { + throw new Error(`data for unexpected bucket: ${row.bucketName}`); + } + start = utils.internalToExternalOpId(startOpId); + } + currentChunk = { + bucket: row.bucketName, + after: start, + has_more: false, + data: [], + next_after: start + }; + } + + const entry = bucketDataRowToOpEntry(row); + currentChunk.data.push(entry); + currentChunk.next_after = entry.op_id; + if (row.targetOp != null && (targetOp == null || row.targetOp > targetOp)) { + targetOp = row.targetOp; + } + + chunkSizeBytes += rowSizeBytes; + batchRowCount++; + } + + if (currentChunk != null) { + currentChunk.has_more = batchRowCount >= batchRowLimit; + yield { chunkData: currentChunk, targetOp }; + } + } + + async getChecksums(checkpoint: bigint, buckets: BucketChecksumRequest[]): Promise { + const result: utils.ChecksumMap = new Map(); + const { db, tables } = this.options.dialect; + for (const bucket of buckets) { + const rows = db + .select() + .from(tables.bucketData) + .where( + and( + eq(tables.bucketData.groupId, this.replicationStreamId), + eq(tables.bucketData.bucketName, bucket.bucket), + lte(tables.bucketData.opId, checkpoint) + ) + ) + .orderBy(asc(tables.bucketData.opId)) + .all(); + const entries = rows.map(bucketDataRowToOpEntry); + const checksum = entries.reduce((total, entry) => utils.addChecksums(total, Number(entry.checksum)), 0); + result.set(bucket.bucket, { + bucket: bucket.bucket, + checksum, + count: entries.length + }); + } + return result; + } + + clearChecksumCache(): void { + // No checksum cache exists in the initial Drizzle storage slice. + } + + private async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters): Promise { + const { db, tables } = this.options.dialect; + const row = db + .select() + .from(tables.writeCheckpoints) + .where( + and( + eq(tables.writeCheckpoints.userId, filters.user_id), + filters.sync_rules_id == null + ? isNull(tables.writeCheckpoints.syncRulesId) + : eq(tables.writeCheckpoints.syncRulesId, filters.sync_rules_id) + ) + ) + .orderBy(desc(tables.writeCheckpoints.checkpoint)) + .limit(1) + .get(); + return row?.checkpoint ?? null; + } + + private async getParameterSets( + checkpoint: bigint, + lookups: sync_rules.ScopedParameterLookup[], + limit: number + ): Promise { + const resultsByLookup = new Map(); + let totalRows = 0; + + for (const lookup of lookups) { + const serializedLookup = storage.serializeLookupBuffer(lookup); + const { db, tables } = this.options.dialect; + const rows = db + .select() + .from(tables.bucketParameters) + .where( + and( + eq(tables.bucketParameters.groupId, this.replicationStreamId), + eq(tables.bucketParameters.lookup, serializedLookup), + lte(tables.bucketParameters.id, checkpoint) + ) + ) + .orderBy(desc(tables.bucketParameters.id)) + .all(); + + const latestBySource = new Map(); + for (const row of rows) { + const key = `${row.sourceTable}:${Buffer.from(row.sourceKey).toString('hex')}`; + if (!latestBySource.has(key)) { + latestBySource.set(key, row); + } + } + + for (const row of latestBySource.values()) { + const parameterRows = parseBucketParameters(row.bucketParameters); + if (parameterRows.length == 0) { + continue; + } + totalRows += parameterRows.length; + if (totalRows > limit) { + throw new storage.ParameterSetLimitExceededError(limit); + } + const existing = resultsByLookup.get(lookup); + if (existing != null) { + existing.push(...parameterRows); + } else { + resultsByLookup.set(lookup, parameterRows); + } + } + } + + const results: sync_rules.ParameterLookupRows[] = []; + resultsByLookup.forEach((rows, lookup) => results.push({ lookup, rows })); + return results; + } + + private async lastManagedWriteCheckpoint(filters: storage.ManagedWriteCheckpointFilters): Promise { + const lsn = filters.heads['1']; + if (lsn == null) { + return null; + } + + const { db, tables } = this.options.dialect; + const rows = db + .select() + .from(tables.writeCheckpoints) + .where(and(eq(tables.writeCheckpoints.userId, filters.user_id), isNull(tables.writeCheckpoints.syncRulesId))) + .orderBy(desc(tables.writeCheckpoints.checkpoint)) + .all(); + + return ( + rows.find((row) => { + const rowHead = getPrimaryReplicationHead(row.heads); + return rowHead != null && rowHead <= lsn; + })?.checkpoint ?? null + ); + } +} + +function getPrimaryReplicationHead(heads: unknown): string | null { + if (heads == null || typeof heads != 'object' || Array.isArray(heads)) { + return null; + } + + const head = (heads as Record)['1']; + return typeof head == 'string' ? head : null; +} + +function parseBucketParameters(value: unknown): sync_rules.SqliteJsonRow[] { + if (typeof value == 'string') { + return JSONBig.parse(value) as sync_rules.SqliteJsonRow[]; + } + return Array.isArray(value) ? (value as sync_rules.SqliteJsonRow[]) : []; +} + +function bucketDataRowToOpEntry(row: BucketDataRow): utils.OplogEntry { + if (row.op == 'PUT' || row.op == 'REMOVE') { + return { + op_id: utils.internalToExternalOpId(row.opId), + op: row.op, + object_type: row.tableName ?? undefined, + object_id: row.rowId ?? undefined, + checksum: Number(row.checksum), + subkey: + row.sourceTable != null && row.sourceKey != null + ? replicaIdToSubkey(row.sourceTable, storage.deserializeReplicaId(Buffer.from(row.sourceKey))) + : undefined, + data: row.op == 'REMOVE' ? null : (row.data ?? undefined) + }; + } + + return { + op_id: utils.internalToExternalOpId(row.opId), + op: row.op as 'CLEAR' | 'MOVE', + checksum: Number(row.checksum) + }; +} + +function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string { + if (storage.isUUID(id)) { + return `${tableId}/${id.toHexString()}`; + } + return uuid.v5(storage.serializeBson({ table: tableId, id }), utils.ID_NAMESPACE); +} diff --git a/modules/module-drizzle-storage/src/storage/storage-index.ts b/modules/module-drizzle-storage/src/storage/storage-index.ts new file mode 100644 index 000000000..5f455092a --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/storage-index.ts @@ -0,0 +1,5 @@ +export * from './DrizzleBucketBatch.js'; +export * from './DrizzleBucketStorageFactory.js'; +export * from './DrizzlePersistedReplicationStream.js'; +export * from './DrizzleStorageProvider.js'; +export * from './DrizzleSyncRulesStorage.js'; diff --git a/modules/module-drizzle-storage/src/storage/unsupported.ts b/modules/module-drizzle-storage/src/storage/unsupported.ts new file mode 100644 index 000000000..7deb9325a --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/unsupported.ts @@ -0,0 +1,3 @@ +export function unsupportedDrizzleStorageFeature(feature: string): never { + throw new Error(`Drizzle bucket storage does not implement ${feature} yet.`); +} diff --git a/modules/module-drizzle-storage/src/types/records.ts b/modules/module-drizzle-storage/src/types/records.ts new file mode 100644 index 000000000..a0a923c38 --- /dev/null +++ b/modules/module-drizzle-storage/src/types/records.ts @@ -0,0 +1,98 @@ +import type { storage } from '@powersync/service-core'; +import type { + BucketDataRow, + BucketParametersRow, + CurrentDataRow, + SourceTableRow, + SyncRulesRow, + WriteCheckpointRow +} from '../drivers/sqlite/schema.js'; + +export interface StorageBucketDataRecord { + id: string; + groupId: number; + bucketName: string; + opId: bigint; + op: string; + sourceTable: string | null; + sourceKey: Buffer | null; + tableName: string | null; + rowId: string | null; + checksum: bigint; + data: string | null; + targetOp: bigint | null; +} + +export interface StorageBucketParametersRecord { + id: bigint; + groupId: number; + sourceTable: string; + sourceKey: Buffer; + lookup: Buffer; + bucketParameters: unknown; +} + +export interface StorageCurrentDataRecord { + id: string; + groupId: number; + sourceTable: string; + sourceKey: Buffer; + buckets: { bucket: string; table: string; id: string }[]; + lookups: string[]; + data: Buffer; + pendingDelete: bigint | null; +} + +export interface StorageSourceTableRecord { + id: string; + groupId: number; + connectionId: number; + relationId: unknown; + schemaName: string; + tableName: string; + replicaIdColumns: unknown; + snapshotDone: boolean; + snapshotTotalEstimatedCount: bigint | null; + snapshotReplicatedCount: bigint | null; + snapshotLastKey: Buffer | null; +} + +export interface StorageSyncRulesRecord { + id: number; + state: storage.SyncRuleState; + snapshotDone: boolean; + snapshotLsn: string | null; + lastCheckpoint: bigint | null; + lastCheckpointLsn: string | null; + noCheckpointBefore: string | null; + slotName: string; + lastCheckpointTs: Date | null; + lastKeepaliveTs: Date | null; + lastFatalError: string | null; + lastFatalErrorTs: Date | null; + keepaliveOp: bigint | null; + storageVersion: number | null; + content: string; + syncPlan: storage.SerializedSyncPlan | null; +} + +export interface StorageWriteCheckpointRecord { + id: string; + syncRulesId: number | null; + userId: string; + checkpoint: bigint; + heads: Record | null; + createdAt: Date; +} + +type Equal = (() => T extends Left ? 1 : 2) extends () => T extends Right ? 1 : 2 ? true : false; +type Assert = T; + +// A future driver gets the same assertions against its own inferred table +// models. This keeps query results identical without sharing table builders. +type _BucketDataMapping = Assert>; +type _BucketParametersMapping = Assert>; +type _CurrentDataMapping = Assert>; +type _SourceTableMapping = Assert>; +type _SyncRulesMapping = Assert>; +type _WriteCheckpointMapping = Assert>; diff --git a/modules/module-drizzle-storage/src/types/types.ts b/modules/module-drizzle-storage/src/types/types.ts new file mode 100644 index 000000000..179278d63 --- /dev/null +++ b/modules/module-drizzle-storage/src/types/types.ts @@ -0,0 +1,37 @@ +import { configFile } from '@powersync/service-types'; +import * as t from 'ts-codec'; + +export const DRIZZLE_SQLITE_STORAGE_TYPE = 'drizzle:sqlite'; + +export const DrizzleSqliteStorageConfig = configFile.BaseStorageConfig.and( + t.object({ + type: t.literal(DRIZZLE_SQLITE_STORAGE_TYPE), + filename: t.string + }) +); + +export type DrizzleSqliteStorageConfig = t.Encoded; +export type DrizzleSqliteStorageConfigDecoded = t.Decoded; +export const DrizzleStorageConfig = DrizzleSqliteStorageConfig; +export type DrizzleStorageConfig = DrizzleSqliteStorageConfig; +export type DrizzleStorageConfigDecoded = DrizzleSqliteStorageConfigDecoded; + +export interface NormalizedDrizzleSqliteStorageConfig { + type: typeof DRIZZLE_SQLITE_STORAGE_TYPE; + filename: string; + max_pool_size: number; +} + +export function isDrizzleStorageConfig(config: configFile.GenericStorageConfig): config is DrizzleSqliteStorageConfig { + return config.type == DRIZZLE_SQLITE_STORAGE_TYPE; +} + +export function normalizeDrizzleSqliteStorageConfig( + config: DrizzleSqliteStorageConfigDecoded +): NormalizedDrizzleSqliteStorageConfig { + return { + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: config.filename, + max_pool_size: config.max_pool_size ?? 10 + }; +} diff --git a/modules/module-drizzle-storage/test/src/migrations.test.ts b/modules/module-drizzle-storage/test/src/migrations.test.ts new file mode 100644 index 000000000..c7681e322 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/migrations.test.ts @@ -0,0 +1,81 @@ +import { Direction } from '@powersync/lib-services-framework'; +import { rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + DRIZZLE_SQLITE_STORAGE_TYPE, + DrizzleMigrationAgent, + createSqliteDrizzleRuntime, + normalizeDrizzleSqliteStorageConfig +} from '../../src/index.js'; + +describe('Drizzle migrations', () => { + const files: string[] = []; + const filename = () => { + const file = join(tmpdir(), `powersync-drizzle-${process.pid}-${Date.now()}-${files.length}.sqlite`); + files.push(file); + return file; + }; + + afterEach(async () => { + await Promise.all( + files + .splice(0) + .flatMap((file) => [ + rm(file, { force: true }), + rm(`${file}-shm`, { force: true }), + rm(`${file}-wal`, { force: true }) + ]) + ); + }); + + it('creates the storage tables and indexes', async () => { + const file = filename(); + await using agent = new DrizzleMigrationAgent({ type: DRIZZLE_SQLITE_STORAGE_TYPE, filename: file }); + await agent.run({ direction: Direction.Up, migrations: [] }); + + const runtime = createSqliteDrizzleRuntime( + normalizeDrizzleSqliteStorageConfig({ + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: file + }) + ); + try { + const names = runtime.client + .prepare(`SELECT name FROM sqlite_master WHERE type IN ('table', 'index')`) + .all() + .map((row: any) => row.name); + expect(names).toEqual( + expect.arrayContaining([ + 'bucket_data', + 'bucket_parameters', + 'current_data', + 'source_tables', + 'sync_rules', + 'write_checkpoints', + 'bucket_data_bucket_op_index', + 'bucket_parameters_lookup_index' + ]) + ); + } finally { + runtime.close(); + } + }); + + it('configures WAL and separate readers for file storage', () => { + const runtime = createSqliteDrizzleRuntime( + normalizeDrizzleSqliteStorageConfig({ + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: filename(), + max_pool_size: 3 + }) + ); + try { + expect(runtime.readers).toHaveLength(2); + expect(runtime.client.pragma('journal_mode', { simple: true })).toBe('wal'); + } finally { + runtime.close(); + } + }); +}); diff --git a/modules/module-drizzle-storage/test/src/setup.ts b/modules/module-drizzle-storage/test/src/setup.ts new file mode 100644 index 000000000..b14ebcec9 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/setup.ts @@ -0,0 +1,11 @@ +import { container } from '@powersync/lib-services-framework'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; +import { beforeAll, beforeEach } from 'vitest'; + +beforeAll(async () => { + container.registerDefaults(); +}); + +beforeEach(async () => { + METRICS_HELPER.resetMetrics(); +}); diff --git a/modules/module-drizzle-storage/test/tsconfig.json b/modules/module-drizzle-storage/test/tsconfig.json new file mode 100644 index 000000000..6538b360a --- /dev/null +++ b/modules/module-drizzle-storage/test/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../../tsconfig.tests.json", + "compilerOptions": { + "declarationDir": "dist/@types", + "tsBuildInfoFile": "dist/.tsbuildinfo", + "lib": ["ES2022", "esnext.disposable"], + "rootDir": "src" + }, + "include": ["src"], + "references": [{ "path": "../" }, { "path": "../../../packages/service-core-tests" }] +} diff --git a/modules/module-drizzle-storage/tsconfig.json b/modules/module-drizzle-storage/tsconfig.json new file mode 100644 index 000000000..9687a076b --- /dev/null +++ b/modules/module-drizzle-storage/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [ + { "path": "../../packages/types" }, + { "path": "../../packages/sync-rules" }, + { "path": "../../packages/service-core" }, + { "path": "../../libs/lib-services" } + ] +} diff --git a/modules/module-drizzle-storage/vitest.config.ts b/modules/module-drizzle-storage/vitest.config.ts new file mode 100644 index 000000000..285c29477 --- /dev/null +++ b/modules/module-drizzle-storage/vitest.config.ts @@ -0,0 +1,7 @@ +import { serviceIntegrationTestConfig } from '../test_config'; + +const config = serviceIntegrationTestConfig(__dirname); +config.test ??= {}; +config.test.testTimeout = 30_000; + +export default config; diff --git a/packages/schema/package.json b/packages/schema/package.json index 84e9d0fb8..f21db8c34 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -33,6 +33,7 @@ "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-mikroorm-storage": "workspace:*", + "@powersync/service-module-drizzle-storage": "workspace:*", "@powersync/service-module-convex": "workspace:*", "@powersync/service-module-mysql": "workspace:*", "@powersync/service-module-mssql": "workspace:*", diff --git a/packages/schema/tsconfig.json b/packages/schema/tsconfig.json index 68c986e19..2910b85b3 100644 --- a/packages/schema/tsconfig.json +++ b/packages/schema/tsconfig.json @@ -23,6 +23,9 @@ { "path": "../../modules/module-mongodb-storage" }, + { + "path": "../../modules/module-drizzle-storage" + }, { "path": "../../modules/module-mysql" }, diff --git a/service/package.json b/service/package.json index 7fe28c327..090424b4b 100644 --- a/service/package.json +++ b/service/package.json @@ -17,6 +17,7 @@ "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-mikroorm-storage": "workspace:*", + "@powersync/service-module-drizzle-storage": "workspace:*", "@powersync/service-module-mssql": "workspace:*", "@powersync/service-module-mysql": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", diff --git a/service/src/util/modules.ts b/service/src/util/modules.ts index c85114fcd..f5d10b980 100644 --- a/service/src/util/modules.ts +++ b/service/src/util/modules.ts @@ -15,6 +15,8 @@ export const DYNAMIC_MODULES: core.ModuleLoaders = { import('@powersync/service-module-mikroorm-storage').then((module) => new module.MikroOrmStorageModule()), 'mikroorm:mysql': () => import('@powersync/service-module-mikroorm-storage').then((module) => new module.MikroOrmStorageModule()), + 'drizzle:sqlite': () => + import('@powersync/service-module-drizzle-storage').then((module) => new module.DrizzleStorageModule()), postgresql: () => import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule()) } diff --git a/service/tsconfig.json b/service/tsconfig.json index 80a86a7b4..245ecc3fa 100644 --- a/service/tsconfig.json +++ b/service/tsconfig.json @@ -33,6 +33,9 @@ { "path": "../modules/module-postgres-storage" }, + { + "path": "../modules/module-drizzle-storage" + }, { "path": "../modules/module-mysql" }, diff --git a/tsconfig.json b/tsconfig.json index 7ad23f53e..bdd14875f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -37,6 +37,9 @@ { "path": "./modules/module-mikroorm-storage" }, + { + "path": "./modules/module-drizzle-storage" + }, { "path": "./modules/module-mssql" }, @@ -103,6 +106,9 @@ { "path": "./modules/module-mikroorm-storage/test" }, + { + "path": "./modules/module-drizzle-storage/test" + }, { "path": "./modules/module-mssql/test" }, From 65e8c46d25bbe8247117f2d4794171d01f989449 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 14 Jul 2026 09:05:09 +0200 Subject: [PATCH 07/11] temporary drizzle dirty --- modules/module-drizzle-storage/README.md | 61 + .../src/storage/DrizzleBucketBatch.ts | 751 ++++ .../src/storage/DrizzleCompactor.ts | 496 +++ .../src/storage/DrizzlePersistedBatch.ts | 421 ++ .../src/__snapshots__/storage.test.ts.snap | 124 + .../__snapshots__/storage_sync.test.ts.snap | 3568 +++++++++++++++++ .../test/src/column-types.test.ts | 47 + .../test/src/storage-provider.test.ts | 34 + .../test/src/storage.test.ts | 21 + .../test/src/storage_bench.test.ts | 18 + .../test/src/storage_sync.test.ts | 14 + .../test/src/sync-rules-storage.test.ts | 175 + .../module-drizzle-storage/test/src/util.ts | 27 + modules/module-postgres/package.json | 1 + modules/module-postgres/test/src/env.ts | 4 +- modules/module-postgres/test/src/util.ts | 32 +- modules/module-postgres/test/tsconfig.json | 3 + .../schema/src/scripts/compile-json-schema.ts | 4 +- pnpm-lock.yaml | 1048 ++++- service/Dockerfile | 4 + 20 files changed, 6827 insertions(+), 26 deletions(-) create mode 100644 modules/module-drizzle-storage/README.md create mode 100644 modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts create mode 100644 modules/module-drizzle-storage/src/storage/DrizzleCompactor.ts create mode 100644 modules/module-drizzle-storage/src/storage/DrizzlePersistedBatch.ts create mode 100644 modules/module-drizzle-storage/test/src/__snapshots__/storage.test.ts.snap create mode 100644 modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap create mode 100644 modules/module-drizzle-storage/test/src/column-types.test.ts create mode 100644 modules/module-drizzle-storage/test/src/storage-provider.test.ts create mode 100644 modules/module-drizzle-storage/test/src/storage.test.ts create mode 100644 modules/module-drizzle-storage/test/src/storage_bench.test.ts create mode 100644 modules/module-drizzle-storage/test/src/storage_sync.test.ts create mode 100644 modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts create mode 100644 modules/module-drizzle-storage/test/src/util.ts diff --git a/modules/module-drizzle-storage/README.md b/modules/module-drizzle-storage/README.md new file mode 100644 index 000000000..6a5710878 --- /dev/null +++ b/modules/module-drizzle-storage/README.md @@ -0,0 +1,61 @@ +# Drizzle Bucket Storage + +Experimental Drizzle-backed bucket storage for the PowerSync service. + +The first implementation uses SQLite through `better-sqlite3`: + +```yaml +storage: + type: drizzle:sqlite + filename: ./powersync-storage.sqlite +``` + +SQLite storage is restricted to the unified runner because checkpoint +notifications are process-local. `:memory:` is supported for tests and +throwaway development. + +## Design + +The SQLite driver owns its Drizzle table declarations. Column codecs map +database values to the shared runtime shapes used by storage code, including +lossless `bigint`, `Buffer`, JSON, boolean, and `Date` values. Compile-time +assertions verify that the inferred SQLite query models match the canonical +storage records. + +Storage classes use Drizzle queries directly. Database-specific behavior is +kept behind the small dialect surface only where required: transactions, +bucket reads, compaction/raw SQL, and checkpoint notification. + +## Migrations + +Drizzle Kit owns schema generation and migration metadata: + +```sh +corepack pnpm --filter @powersync/service-module-drizzle-storage drizzle:generate:sqlite +corepack pnpm --filter @powersync/service-module-drizzle-storage drizzle:check:sqlite +``` + +Generated SQL and metadata are committed under `src/migrations/sqlite` and +copied into the built package. Runtime migrations are exposed through the +normal PowerSync migration agent and guarded by a SQLite-backed lock. + +## Development + +```sh +corepack pnpm --filter @powersync/service-module-drizzle-storage build +corepack pnpm --filter @powersync/service-module-drizzle-storage build:tests +corepack pnpm --filter @powersync/service-module-drizzle-storage test --run +``` + +PostgreSQL replication tests can opt into this storage with: + +```sh +TEST_MONGO_STORAGE=false \ +TEST_POSTGRES_STORAGE=false \ +TEST_MIKROORM_SQLITE_STORAGE=false \ +TEST_DRIZZLE_SQLITE_STORAGE=true \ +corepack pnpm --filter @powersync/service-module-postgres test test/src/wal_stream.test.ts --run +``` + +Set `DRIZZLE_SQLITE_STORAGE_TEST_FILENAME` to override the file-backed test +database path. diff --git a/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts b/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts new file mode 100644 index 000000000..81adf952a --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts @@ -0,0 +1,751 @@ +import { BaseObserver, DO_NOT_LOG, Logger, ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { ColumnDescriptor, storage, utils } from '@powersync/service-core'; +import * as sync_rules from '@powersync/service-sync-rules'; +import { and, count, desc, eq, inArray, isNotNull, isNull, lte, ne } from 'drizzle-orm'; +import * as uuid from 'uuid'; +import type { SourceTableRow } from '../drivers/sqlite/schema.js'; +import type { DrizzleStorageTransaction } from '../drivers/sqlite/sqlite-config.js'; +import { DrizzleBucketStorageFactory } from './DrizzleBucketStorageFactory.js'; +import { currentBuckets, currentLookups, DrizzlePersistedBatch } from './DrizzlePersistedBatch.js'; +import { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +export interface DrizzleBucketBatchOptions { + factory: DrizzleBucketStorageFactory; + dialect: DrizzleStorageDialect; + logger: Logger; + syncRules: sync_rules.HydratedSyncConfig; + replicationStreamId: number; + replicationStreamName: string; + lastCheckpointLsn: string | null; + keepaliveOp: bigint | null; + resumeFromLsn: string | null; + storeCurrentData: boolean; + skipExistingRows: boolean; + markRecordUnavailable: storage.BucketStorageMarkRecordUnavailable | undefined; + hooks: storage.StorageHooks | undefined; +} + +const MAX_OPERATION_BATCH_COUNT = 2_000; + +export class DrizzleBucketBatch + extends BaseObserver + implements storage.BucketStorageBatch +{ + [DO_NOT_LOG] = true; + + public last_flushed_op: bigint | null = null; + public resumeFromLsn: string | null; + public readonly skipExistingRows: boolean; + + private lastCheckpointLsnValue: string | null; + private persistedOp: bigint | null; + private readonly pendingOperations: storage.SaveOptions[] = []; + private readonly customWriteCheckpointBatch: storage.CustomWriteCheckpointOptions[] = []; + private needsActivation = true; + + constructor(private readonly options: DrizzleBucketBatchOptions) { + super(); + this.lastCheckpointLsnValue = options.lastCheckpointLsn; + this.resumeFromLsn = options.resumeFromLsn; + this.skipExistingRows = options.skipExistingRows; + this.persistedOp = options.keepaliveOp; + } + + get lastCheckpointLsn(): string | null { + return this.lastCheckpointLsnValue; + } + + async [Symbol.asyncDispose](): Promise { + if (this.customWriteCheckpointBatch.length > 0) { + this.options.logger.warn('Disposing writer with unflushed custom write checkpoints'); + } + super.clearListeners(); + } + + async dispose(): Promise { + await this[Symbol.asyncDispose](); + } + + async resolveTables(options: storage.ResolveTablesOptions): Promise { + const syncRules = options.syncRules ?? this.options.syncRules; + const { connection_id, source } = options; + const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source; + const normalizedReplicaIdColumns = normalizeReplicaIdColumns(replicaIdColumns); + const relationId = { object_id: objectId }; + const { tables } = this.options.dialect; + + return this.options.dialect.transaction((tx) => { + const existingRows = tx + .select() + .from(tables.sourceTables) + .where( + and( + eq(tables.sourceTables.groupId, this.options.replicationStreamId), + eq(tables.sourceTables.connectionId, connection_id) + ) + ) + .all(); + + let sourceTableRow = + existingRows.find((row) => { + const matchesRelationId = objectId == null || relationObjectId(row.relationId) == objectId; + return ( + row.schemaName == schema && + row.tableName == table && + matchesRelationId && + jsonEquals(row.replicaIdColumns, normalizedReplicaIdColumns) + ); + }) ?? null; + + if (sourceTableRow == null) { + sourceTableRow = tx + .insert(tables.sourceTables) + .values({ + id: options.idGenerator ? String(options.idGenerator()) : uuid.v4(), + groupId: this.options.replicationStreamId, + connectionId: connection_id, + relationId, + schemaName: schema, + tableName: table, + replicaIdColumns: normalizedReplicaIdColumns, + snapshotDone: false, + snapshotTotalEstimatedCount: null, + snapshotReplicatedCount: null, + snapshotLastKey: null + }) + .returning() + .get(); + } + + const sourceTable = sourceTableFromRow(sourceTableRow, connectionTag, syncRules); + sourceTable.storeCurrentData = sendsCompleteRows !== true; + + const dropTables = existingRows + .filter((row) => row.id != sourceTableRow.id) + .filter((row) => { + const matchesTableName = row.schemaName == schema && row.tableName == table; + return objectId == null ? matchesTableName : relationObjectId(row.relationId) == objectId || matchesTableName; + }) + .map((row) => sourceTableFromRow(row, connectionTag, syncRules)); + + return { + tables: [sourceTable], + dropTables + }; + }); + } + + async getSourceTableStatus(table: storage.SourceTable): Promise { + const { db, tables } = this.options.dialect; + const row = db + .select() + .from(tables.sourceTables) + .where( + and( + eq(tables.sourceTables.groupId, this.options.replicationStreamId), + eq(tables.sourceTables.id, String(table.id)) + ) + ) + .get(); + + return row == null ? null : sourceTableFromRow(row, table.ref.connectionTag, this.options.syncRules); + } + + async save(record: storage.SaveOptions): Promise { + const { after, before, sourceTable, tag } = record; + const storeCurrentData = this.options.storeCurrentData && sourceTable.storeCurrentData; + for (const event of this.getTableEvents(sourceTable)) { + this.iterateListeners((cb) => + cb.replicationEvent?.({ + batch: this, + table: sourceTable, + data: { + op: tag, + after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined, + before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined + }, + event + }) + ); + } + + if (!sourceTable.syncData && !sourceTable.syncParameters) { + return null; + } + + this.pendingOperations.push(record); + if (this.pendingOperations.length >= MAX_OPERATION_BATCH_COUNT) { + return this.flush(); + } + return null; + } + + async truncate(sourceTables: storage.SourceTable[]): Promise { + await this.flush(); + + let nextOpId = this.getNextOpId(); + const firstOpId = nextOpId; + + this.options.dialect.transaction((tx) => { + const persistedBatch = this.createPersistedBatch(tx); + for (const table of sourceTables) { + if (!table.syncData && !table.syncParameters) { + continue; + } + + const rows = tx + .select() + .from(this.options.dialect.tables.currentData) + .where( + and( + eq(this.options.dialect.tables.currentData.groupId, this.options.replicationStreamId), + eq(this.options.dialect.tables.currentData.sourceTable, String(table.id)) + ) + ) + .all(); + + for (const row of rows) { + const sourceKey = storage.deserializeReplicaId(Buffer.from(row.sourceKey)); + if (table.syncData) { + nextOpId = persistedBatch.persistBucketData({ + table, + sourceKey, + existingBuckets: currentBuckets(row), + evaluated: [], + nextOpId + }); + } + + if (table.syncParameters) { + nextOpId = persistedBatch.persistParameterData({ + table, + sourceKey, + existingLookups: currentLookups(row), + evaluated: [], + nextOpId + }); + } + } + tx.delete(this.options.dialect.tables.currentData) + .where( + and( + eq(this.options.dialect.tables.currentData.groupId, this.options.replicationStreamId), + eq(this.options.dialect.tables.currentData.sourceTable, String(table.id)) + ) + ) + .run(); + } + }); + + if (nextOpId == firstOpId) { + return null; + } + + const lastOpId = nextOpId - 1n; + this.persistedOp = lastOpId; + this.last_flushed_op = lastOpId; + this.logFlushedOperations({ + operationCount: sourceTables.length, + firstOpId, + nextOpId, + operationLabel: 'truncate source table' + }); + return { flushed_op: lastOpId }; + } + + async drop(sourceTables: storage.SourceTable[]): Promise { + const result = await this.truncate(sourceTables); + this.options.dialect.transaction((tx) => { + for (const table of sourceTables) { + tx.delete(this.options.dialect.tables.sourceTables) + .where( + and( + eq(this.options.dialect.tables.sourceTables.groupId, this.options.replicationStreamId), + eq(this.options.dialect.tables.sourceTables.id, String(table.id)) + ) + ) + .run(); + } + }); + return result; + } + + async flush(_options?: storage.BatchBucketFlushOptions): Promise { + let result: storage.FlushedResult | null = null; + if (this.pendingOperations.length > 0) { + await this.options.hooks?.beforeBatchFlush?.(this); + const operations = this.pendingOperations.splice(0); + let nextOpId = this.getNextOpId(); + const firstOpId = nextOpId; + for (const batch of chunked(operations, MAX_OPERATION_BATCH_COUNT)) { + this.options.dialect.transaction((tx) => { + const persistedBatch = this.createPersistedBatch(tx); + nextOpId = persistedBatch.persistOperations(batch, nextOpId); + }); + } + const lastOpId = nextOpId - 1n; + this.persistedOp = lastOpId; + this.last_flushed_op = lastOpId; + result = { flushed_op: lastOpId }; + this.logFlushedOperations({ + operationCount: operations.length, + firstOpId, + nextOpId + }); + await this.options.hooks?.afterBatchFlush?.(this); + } + + await this.flushCustomWriteCheckpoints(); + return result; + } + + async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { + await this.flush(); + const createEmptyCheckpoints = options?.createEmptyCheckpoints ?? true; + const now = new Date(); + const { tables } = this.options.dialect; + + const result = this.options.dialect.transaction((tx) => { + const syncRulesRow = tx + .select() + .from(tables.syncRules) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .get(); + if (syncRulesRow == null) { + throw new Error(`Missing replication stream ${this.options.replicationStreamId}`); + } + + const canCheckpoint = + syncRulesRow.snapshotDone === true && + (syncRulesRow.lastCheckpointLsn == null || syncRulesRow.lastCheckpointLsn <= lsn) && + (syncRulesRow.noCheckpointBefore == null || syncRulesRow.noCheckpointBefore <= lsn); + + let checkpointCreated = false; + + if (canCheckpoint) { + const newLastCheckpoint = maxBigint( + syncRulesRow.lastCheckpoint, + this.persistedOp, + syncRulesRow.keepaliveOp, + 0n + ); + const changed = syncRulesRow.lastCheckpoint !== newLastCheckpoint || syncRulesRow.keepaliveOp != null; + + if (changed || createEmptyCheckpoints) { + tx.update(tables.syncRules) + .set({ + lastCheckpointLsn: lsn, + lastCheckpointTs: now, + lastKeepaliveTs: now, + lastFatalError: null, + keepaliveOp: null, + lastCheckpoint: newLastCheckpoint, + snapshotLsn: null + }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + checkpointCreated = true; + tx.delete(tables.currentData) + .where( + and( + eq(tables.currentData.groupId, this.options.replicationStreamId), + lte(tables.currentData.pendingDelete, newLastCheckpoint) + ) + ) + .run(); + } else { + tx.update(tables.syncRules) + .set({ lastKeepaliveTs: now }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + } + } else { + tx.update(tables.syncRules) + .set({ + keepaliveOp: maxBigint(syncRulesRow.keepaliveOp, this.persistedOp, 0n), + lastKeepaliveTs: now + }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + } + + return { + checkpointBlocked: !canCheckpoint, + checkpointCreated + }; + }); + + if (!result.checkpointBlocked) { + await this.autoActivate(lsn); + } + + this.persistedOp = null; + this.lastCheckpointLsnValue = lsn; + this.options.factory.checkpointWatcher.notify(); + return result; + } + + keepalive(lsn: string): Promise { + return this.commit(lsn, { createEmptyCheckpoints: true }); + } + + async setResumeLsn(lsn: string): Promise { + const { db, tables } = this.options.dialect; + const result = db + .update(tables.syncRules) + .set({ snapshotLsn: lsn }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + if (result.changes == 0) { + throw new Error(`Missing replication stream ${this.options.replicationStreamId}`); + } + this.resumeFromLsn = lsn; + } + + async markAllSnapshotDone(noCheckpointBeforeLsn: string): Promise { + await this.markSnapshotDoneInternal(noCheckpointBeforeLsn); + } + + async markSnapshotDone(noCheckpointBeforeLsn: string, options?: { throwOnConflict?: boolean }): Promise { + const { db, tables } = this.options.dialect; + const remaining = + db + .select({ value: count() }) + .from(tables.sourceTables) + .where( + and( + eq(tables.sourceTables.groupId, this.options.replicationStreamId), + eq(tables.sourceTables.snapshotDone, false) + ) + ) + .get()?.value ?? 0; + + if (remaining > 0) { + if (options?.throwOnConflict ?? true) { + throw new ReplicationAssertionError( + `Cannot mark snapshot done while ${remaining} source table${remaining == 1 ? '' : 's'} still require snapshotting` + ); + } + return; + } + + await this.markSnapshotDoneInternal(noCheckpointBeforeLsn); + } + + async markTableSnapshotRequired(_table: storage.SourceTable): Promise { + const { db, tables } = this.options.dialect; + const result = db + .update(tables.syncRules) + .set({ snapshotDone: false }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + if (result.changes == 0) { + throw new Error(`Missing replication stream ${this.options.replicationStreamId}`); + } + } + + async markTableSnapshotDone( + tables: storage.SourceTable[], + noCheckpointBeforeLsn?: string + ): Promise { + const ids = tables.map((table) => String(table.id)); + this.options.dialect.transaction((tx) => { + if (ids.length > 0) { + tx.update(this.options.dialect.tables.sourceTables) + .set({ + snapshotDone: true, + snapshotTotalEstimatedCount: null, + snapshotReplicatedCount: null, + snapshotLastKey: null + }) + .where(inArray(this.options.dialect.tables.sourceTables.id, ids)) + .run(); + } + + if (noCheckpointBeforeLsn != null) { + this.assignNoCheckpointBefore(tx, noCheckpointBeforeLsn); + } + }); + + return tables.map((table) => { + const copy = table.clone(); + copy.snapshotComplete = true; + copy.snapshotStatus = undefined; + return copy; + }); + } + + async updateTableProgress( + table: storage.SourceTable, + progress: Partial + ): Promise { + const copy = table.clone(); + const snapshotStatus = { + totalEstimatedCount: progress.totalEstimatedCount ?? copy.snapshotStatus?.totalEstimatedCount ?? 0, + replicatedCount: progress.replicatedCount ?? copy.snapshotStatus?.replicatedCount ?? 0, + lastKey: progress.lastKey ?? copy.snapshotStatus?.lastKey ?? null + }; + copy.snapshotStatus = snapshotStatus; + + const { db, tables } = this.options.dialect; + const result = db + .update(tables.sourceTables) + .set({ + snapshotTotalEstimatedCount: BigInt(snapshotStatus.totalEstimatedCount), + snapshotReplicatedCount: BigInt(snapshotStatus.replicatedCount), + snapshotLastKey: snapshotStatus.lastKey == null ? null : Buffer.from(snapshotStatus.lastKey) + }) + .where(eq(tables.sourceTables.id, String(table.id))) + .run(); + if (result.changes == 0) { + throw new Error(`Missing source table ${table.id}`); + } + + return copy; + } + + addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void { + this.customWriteCheckpointBatch.push({ + ...checkpoint, + sync_rules_id: this.options.replicationStreamId + }); + } + + private async flushCustomWriteCheckpoints(): Promise { + if (this.customWriteCheckpointBatch.length == 0) { + return; + } + + const batch = this.customWriteCheckpointBatch.splice(0); + this.options.dialect.transaction((tx) => { + for (const checkpoint of batch) { + const table = this.options.dialect.tables.writeCheckpoints; + const existing = tx + .select() + .from(table) + .where( + and( + eq(table.userId, checkpoint.user_id), + checkpoint.sync_rules_id == null + ? isNull(table.syncRulesId) + : eq(table.syncRulesId, checkpoint.sync_rules_id) + ) + ) + .get(); + + if (existing == null) { + tx.insert(table) + .values({ + id: uuid.v4(), + syncRulesId: checkpoint.sync_rules_id, + userId: checkpoint.user_id, + checkpoint: checkpoint.checkpoint, + heads: null, + createdAt: new Date() + }) + .run(); + } else { + tx.update(table) + .set({ + checkpoint: checkpoint.checkpoint, + createdAt: new Date() + }) + .where(eq(table.id, existing.id)) + .run(); + } + } + }); + this.options.factory.checkpointWatcher.notify(); + } + + private createPersistedBatch(tx: DrizzleStorageTransaction): DrizzlePersistedBatch { + return new DrizzlePersistedBatch({ + tx, + dialect: this.options.dialect, + logger: this.options.logger, + syncRules: this.options.syncRules, + replicationStreamId: this.options.replicationStreamId, + storeCurrentData: this.options.storeCurrentData, + skipExistingRows: this.options.skipExistingRows, + markRecordUnavailable: this.options.markRecordUnavailable + }); + } + + private getNextOpId(): bigint { + const { db, tables } = this.options.dialect; + const bucketData = db + .select({ value: tables.bucketData.opId }) + .from(tables.bucketData) + .orderBy(desc(tables.bucketData.opId)) + .limit(1) + .get(); + const bucketParameters = db + .select({ value: tables.bucketParameters.id }) + .from(tables.bucketParameters) + .orderBy(desc(tables.bucketParameters.id)) + .limit(1) + .get(); + const pendingDelete = db + .select({ value: tables.currentData.pendingDelete }) + .from(tables.currentData) + .where(isNotNull(tables.currentData.pendingDelete)) + .orderBy(desc(tables.currentData.pendingDelete)) + .limit(1) + .get(); + + return maxBigint(bucketData?.value, bucketParameters?.value, pendingDelete?.value, 0n) + 1n; + } + + private logFlushedOperations(options: { + operationCount: number; + firstOpId: bigint; + nextOpId: bigint; + operationLabel?: string; + }): void { + const storageOperationCount = options.nextOpId - options.firstOpId; + const operationLabel = options.operationLabel ?? 'source operation'; + const pluralizedOperationLabel = options.operationCount == 1 ? operationLabel : `${operationLabel}s`; + + if (storageOperationCount == 0n) { + this.options.logger.info( + `[${this.options.replicationStreamName}] Flushed ${options.operationCount} ${pluralizedOperationLabel} to Drizzle storage DB with no new storage ops` + ); + return; + } + + this.options.logger.info( + `[${this.options.replicationStreamName}] Flushed ${options.operationCount} ${pluralizedOperationLabel} to Drizzle storage DB as ${storageOperationCount.toString()} storage ops (${options.firstOpId.toString()}-${(options.nextOpId - 1n).toString()})` + ); + } + + private async markSnapshotDoneInternal(noCheckpointBeforeLsn: string): Promise { + this.options.dialect.transaction((tx) => { + this.assignNoCheckpointBefore(tx, noCheckpointBeforeLsn); + }); + this.options.factory.checkpointWatcher.notify(); + } + + private assignNoCheckpointBefore(tx: DrizzleStorageTransaction, noCheckpointBeforeLsn: string): void { + const table = this.options.dialect.tables.syncRules; + const row = tx.select().from(table).where(eq(table.id, this.options.replicationStreamId)).get(); + if (row == null) { + throw new Error(`Missing replication stream ${this.options.replicationStreamId}`); + } + tx.update(table) + .set({ + snapshotDone: true, + lastKeepaliveTs: new Date(), + noCheckpointBefore: + row.noCheckpointBefore == null || row.noCheckpointBefore < noCheckpointBeforeLsn + ? noCheckpointBeforeLsn + : row.noCheckpointBefore + }) + .where(eq(table.id, this.options.replicationStreamId)) + .run(); + } + + private async autoActivate(lsn: string): Promise { + if (!this.needsActivation) { + return; + } + + let didActivate = false; + this.options.dialect.transaction((tx) => { + const table = this.options.dialect.tables.syncRules; + const syncRulesRow = tx.select().from(table).where(eq(table.id, this.options.replicationStreamId)).get(); + + if (syncRulesRow?.state == storage.SyncRuleState.PROCESSING && syncRulesRow.snapshotDone) { + tx.update(table) + .set({ state: storage.SyncRuleState.ACTIVE }) + .where(eq(table.id, this.options.replicationStreamId)) + .run(); + tx.update(table) + .set({ state: storage.SyncRuleState.STOP }) + .where( + and( + ne(table.id, this.options.replicationStreamId), + inArray(table.state, [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED]) + ) + ) + .run(); + didActivate = true; + this.needsActivation = false; + } else if (syncRulesRow?.state != storage.SyncRuleState.PROCESSING) { + this.needsActivation = false; + } + }); + + if (didActivate) { + this.options.logger.info(`Activated new replication stream at ${lsn}`); + } + } + + private getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] { + return this.options.syncRules.eventDescriptors.filter((event) => + [...event.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref)) + ); + } +} + +function sourceTableFromRow( + row: SourceTableRow, + connectionTag: string, + syncRules: sync_rules.HydratedSyncConfig +): storage.SourceTable { + const ref = { connectionTag, schema: row.schemaName, name: row.tableName }; + const sourceTable = new storage.SourceTable({ + id: row.id, + ref, + objectId: relationObjectId(row.relationId), + replicaIdColumns: replicaIdColumns(row.replicaIdColumns), + snapshotComplete: row.snapshotDone ?? true, + ...syncRules.getMatchingSources(ref) + }); + + if (!sourceTable.snapshotComplete) { + sourceTable.snapshotStatus = { + totalEstimatedCount: Number(row.snapshotTotalEstimatedCount ?? -1n), + replicatedCount: Number(row.snapshotReplicatedCount ?? 0n), + lastKey: row.snapshotLastKey ?? null + }; + } + + sourceTable.syncEvent = syncRules.tableTriggersEvent(ref); + sourceTable.syncData = sourceTable.bucketDataSources.length > 0; + sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0; + return sourceTable; +} + +function normalizeReplicaIdColumns(replicaIdColumns: ColumnDescriptor[]): ColumnDescriptor[] { + return replicaIdColumns.map((column) => ({ + name: column.name, + type: column.type, + typeId: typeof column.typeId === 'undefined' ? column.typeId : Number(column.typeId) + })); +} + +function replicaIdColumns(value: unknown): ColumnDescriptor[] { + return Array.isArray(value) ? (value as ColumnDescriptor[]) : []; +} + +function relationObjectId(value: unknown): string | number | undefined { + if (value == null || typeof value != 'object' || Array.isArray(value)) { + return undefined; + } + const objectId = (value as Record).object_id; + return typeof objectId == 'string' || typeof objectId == 'number' ? objectId : undefined; +} + +function jsonEquals(left: unknown, right: unknown): boolean { + return JSON.stringify(left ?? null) == JSON.stringify(right ?? null); +} + +function maxBigint(...values: (bigint | null | undefined)[]): bigint { + return values.reduce((max, value) => (value != null && value > max ? value : max), 0n); +} + +function* chunked(items: T[], size: number): Generator { + for (let index = 0; index < items.length; index += size) { + yield items.slice(index, index + size); + } +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleCompactor.ts b/modules/module-drizzle-storage/src/storage/DrizzleCompactor.ts new file mode 100644 index 000000000..eb6b4ab24 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleCompactor.ts @@ -0,0 +1,496 @@ +import { Logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import * as uuid from 'uuid'; +import type { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +interface CurrentBucketState { + bucket: string; + seen: Map; + trackingSize: number; + lastNotPut: bigint | null; + opsSincePut: number; +} + +interface ParameterCompactionRow { + id: bigint; + sourceTable: string; + sourceKey: Buffer; + lookup: Buffer; + bucketParameters: unknown; +} + +interface RawParameterCompactionRow { + id: bigint | number | string; + source_table?: string; + sourceTable?: string; + source_key?: Buffer | Uint8Array; + sourceKey?: Buffer | Uint8Array; + lookup: Buffer | Uint8Array; + bucket_parameters?: unknown; + bucketParameters?: unknown; +} + +interface RawBucketDataRow { + id: string; + bucket_name: string; + op_id: bigint | number | string; + op: string; + source_table: string | null; + source_key: Buffer | Uint8Array | null; + table_name: string | null; + row_id: string | null; + checksum: bigint | number | string; + target_op: bigint | number | string | null; +} + +interface CompactionBucketDataRow { + id: string; + bucketName: string; + opId: bigint; + op: string; + sourceTable: string | null; + sourceKey: Buffer | null; + tableName: string | null; + rowId: string | null; + checksum: bigint; + targetOp: bigint | null; +} + +export interface DrizzleCompactOptions extends storage.CompactOptions { + logger: Logger; +} + +const BIGINT_MAX = 9223372036854775807n; +const DEFAULT_CLEAR_BATCH_LIMIT = 5000; +const DEFAULT_MOVE_BATCH_LIMIT = 2000; +const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; +const DEFAULT_MEMORY_LIMIT_MB = 64; +const PARAMETER_DELETE_BATCH_LIMIT = 1000; +const PARAMETER_SCAN_BATCH_LIMIT = 10_000; + +export class DrizzleCompactor { + private readonly idLimitBytes: number; + private readonly moveBatchLimit: number; + private readonly moveBatchQueryLimit: number; + private readonly clearBatchLimit: number; + private readonly maxOpId: bigint; + private readonly buckets: string[] | undefined; + private readonly logger: Logger; + + private pendingMoves: { id: string; targetOp: bigint }[] = []; + + constructor( + private readonly dialect: DrizzleStorageDialect, + private readonly groupId: number, + options: DrizzleCompactOptions + ) { + this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; + this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; + this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; + this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; + this.maxOpId = options.maxOpId ?? 0n; + this.buckets = options.compactBuckets; + this.logger = options.logger; + } + + async compact(): Promise { + if (this.maxOpId <= 0n) { + return; + } + + if (this.buckets != null) { + for (const bucket of this.buckets) { + await this.compactSingleBucket(bucket); + } + } else { + await this.compactAllBuckets(); + } + } + + async compactParameterData(options: storage.CompactOptions): Promise { + if (this.maxOpId <= 0n) { + return; + } + + const lastByKey = new Map(); + const removeIds = new Set(); + const keysWithDuplicateValues = new Set(); + const maxCacheSize = options.compactParameterCacheLimit ?? 10_000; + const flushDeletes = async (force: boolean) => { + if (removeIds.size < PARAMETER_DELETE_BATCH_LIMIT && !(force && removeIds.size > 0)) { + return; + } + + const removedCount = removeIds.size; + for (const id of removeIds) { + await this.executeRun( + ` + DELETE FROM bucket_parameters + WHERE group_id = ? AND id = ? + `, + [this.groupId, id.toString()] + ); + } + removeIds.clear(); + this.logger.info(`Removed ${removedCount} compacted parameter entries`); + }; + + for await (const row of this.streamParameterRows()) { + const key = parameterKey(row); + const previous = lastByKey.get(key); + if (previous != null && sameBucketParameters(previous.bucketParameters, row.bucketParameters)) { + removeIds.add(row.id); + keysWithDuplicateValues.add(key); + } + + if (isEmptyBucketParameters(row.bucketParameters) && row.id < this.maxOpId && !keysWithDuplicateValues.has(key)) { + await flushDeletes(true); + const candidate = lastByKey.get(key); + await this.executeRun( + ` + DELETE FROM bucket_parameters + WHERE group_id = ? + AND lookup = ? + AND source_table = ? + AND source_key = ? + AND id <= ? + `, + [this.groupId, row.lookup, row.sourceTable, row.sourceKey, row.id.toString()] + ); + if (candidate != null && candidate.id <= row.id) { + removeIds.add(candidate.id); + } + removeIds.add(row.id); + lastByKey.delete(key); + } else { + lastByKey.set(key, row); + } + + if (lastByKey.size > maxCacheSize) { + const oldest = lastByKey.keys().next().value; + if (oldest != null) { + lastByKey.delete(oldest); + } + } + + await flushDeletes(false); + } + await flushDeletes(true); + lastByKey.clear(); + } + + private async *streamParameterRows(): AsyncIterable { + let lastRow: ParameterCompactionRow | null = null; + + while (true) { + const params: unknown[] = [this.groupId, this.maxOpId]; + let cursorFilter = ''; + if (lastRow != null) { + cursorFilter = ` + AND ( + lookup > ? + OR (lookup = ? AND source_table > ?) + OR (lookup = ? AND source_table = ? AND source_key > ?) + OR (lookup = ? AND source_table = ? AND source_key = ? AND id > ?) + ) + `; + params.push( + lastRow.lookup, + lastRow.lookup, + lastRow.sourceTable, + lastRow.lookup, + lastRow.sourceTable, + lastRow.sourceKey, + lastRow.lookup, + lastRow.sourceTable, + lastRow.sourceKey, + lastRow.id + ); + } + params.push(PARAMETER_SCAN_BATCH_LIMIT); + + const rows = await this.executeAll( + ` + SELECT id, source_table, source_key, lookup, bucket_parameters + FROM bucket_parameters + WHERE group_id = ? + AND id <= ? + ${cursorFilter} + ORDER BY lookup ASC, source_table ASC, source_key ASC, id ASC + LIMIT ? + `, + params + ); + + if (rows.length == 0) { + return; + } + + for (const rawRow of rows) { + lastRow = rawParameterRow(rawRow); + yield lastRow; + } + } + } + + private async compactAllBuckets(): Promise { + const discoveryBatchSize = 200; + let lastBucket = ''; + + while (true) { + const rows = await this.executeAll<{ bucket_name: string }>( + ` + SELECT DISTINCT bucket_name + FROM bucket_data + WHERE group_id = ? AND bucket_name > ? + ORDER BY bucket_name ASC + LIMIT ? + `, + [this.groupId, lastBucket, discoveryBatchSize] + ); + if (rows.length == 0) { + break; + } + + for (const row of rows) { + await this.compactSingleBucket(row.bucket_name); + } + lastBucket = rows[rows.length - 1].bucket_name; + } + } + + private async compactSingleBucket(bucket: string): Promise { + const currentState: CurrentBucketState = { + bucket, + seen: new Map(), + trackingSize: 0, + lastNotPut: null, + opsSincePut: 0 + }; + let upperOpIdLimit = BIGINT_MAX; + + while (true) { + const batch = ( + await this.executeAll( + ` + SELECT id, bucket_name, op_id, op, source_table, source_key, table_name, row_id, checksum, target_op + FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id < ? + AND op_id <= ? + ORDER BY op_id DESC + LIMIT ? + `, + [this.groupId, bucket, upperOpIdLimit, this.maxOpId, this.moveBatchQueryLimit] + ) + ).map(rawBucketDataRow); + + if (batch.length == 0) { + break; + } + + upperOpIdLimit = batch[batch.length - 1].opId; + + for (const row of batch) { + let isPersistentPut = row.op == 'PUT'; + + if (row.op == 'REMOVE' || row.op == 'PUT') { + const key = compactBucketRowKey(row); + const targetOp = currentState.seen.get(utils.flatstr(key)); + if (targetOp != null) { + isPersistentPut = false; + this.pendingMoves.push({ id: row.id, targetOp }); + } else if (currentState.trackingSize < this.idLimitBytes) { + currentState.seen.set(utils.flatstr(key), row.opId); + currentState.trackingSize += key.length + 140; + } + } + + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (row.op != 'CLEAR') { + currentState.lastNotPut ??= row.opId; + currentState.opsSincePut += 1; + } + + if (this.pendingMoves.length >= this.moveBatchLimit) { + await this.flushMoves(); + } + } + } + + await this.flushMoves(); + currentState.seen.clear(); + if (currentState.lastNotPut != null && currentState.opsSincePut > 1) { + this.logger.info( + `Inserting CLEAR at ${this.groupId}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + await this.clearBucket(currentState.bucket, currentState.lastNotPut); + } + } + + private async flushMoves(): Promise { + if (this.pendingMoves.length == 0) { + return; + } + + const batch = this.pendingMoves.splice(0); + this.logger.info(`Compacting ${batch.length} ops`); + for (const { id, targetOp } of batch) { + await this.executeRun( + ` + UPDATE bucket_data + SET op = 'MOVE', + target_op = ?, + table_name = NULL, + row_id = NULL, + data = NULL, + source_table = NULL, + source_key = NULL + WHERE id = ? + `, + [targetOp, id] + ); + } + } + + private async clearBucket(bucket: string, op: bigint): Promise { + let done = false; + while (!done) { + const operationRows = ( + await this.executeAll( + ` + SELECT id, bucket_name, op_id, op, source_table, source_key, table_name, row_id, checksum, target_op + FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id <= ? + ORDER BY op_id ASC + LIMIT ? + `, + [this.groupId, bucket, op, this.clearBatchLimit] + ) + ).map(rawBucketDataRow); + + let checksum = 0; + let lastOpId: bigint | null = null; + let targetOp: bigint | null = null; + let gotAnOp = false; + + for (const operation of operationRows) { + if (operation.op != 'MOVE' && operation.op != 'REMOVE' && operation.op != 'CLEAR') { + throw new Error(`Unexpected ${operation.op} operation at ${this.groupId}:${bucket}:${operation.opId}`); + } + + checksum = utils.addChecksums(checksum, Number(operation.checksum)); + lastOpId = operation.opId; + if (operation.op != 'CLEAR') { + gotAnOp = true; + } + if (operation.targetOp != null && (targetOp == null || operation.targetOp > targetOp)) { + targetOp = operation.targetOp; + } + } + + if (!gotAnOp || lastOpId == null) { + done = true; + return; + } + + this.logger.info(`Flushing CLEAR at ${lastOpId}`); + await this.executeRun( + ` + DELETE FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id <= ? + `, + [this.groupId, bucket, lastOpId] + ); + await this.executeRun( + ` + INSERT INTO bucket_data ( + id, + group_id, + bucket_name, + op_id, + op, + checksum, + target_op + ) VALUES (?, ?, ?, ?, 'CLEAR', ?, ?) + `, + [uuid.v4(), this.groupId, bucket, lastOpId, BigInt(checksum), targetOp] + ); + } + } + + private async executeAll(sql: string, params: unknown[]): Promise { + return this.dialect.db.$client.prepare(sql).all(...params) as T[]; + } + + private async executeRun(sql: string, params: unknown[]): Promise { + this.dialect.db.$client.prepare(sql).run(...params); + } +} + +function compactBucketRowKey(row: CompactionBucketDataRow): string { + return `${row.tableName}/${row.rowId}/${row.sourceTable}.${row.sourceKey == null ? '' : row.sourceKey.toString('base64')}`; +} + +function rawBucketDataRow(row: RawBucketDataRow): CompactionBucketDataRow { + return { + id: row.id, + bucketName: row.bucket_name, + opId: BigInt(row.op_id), + op: row.op, + sourceTable: row.source_table, + sourceKey: row.source_key == null ? null : Buffer.from(row.source_key), + tableName: row.table_name, + rowId: row.row_id, + checksum: BigInt(row.checksum), + targetOp: row.target_op == null ? null : BigInt(row.target_op) + }; +} + +function rawParameterRow(row: RawParameterCompactionRow): ParameterCompactionRow { + const sourceTable = row.source_table ?? row.sourceTable; + const sourceKey = row.source_key ?? row.sourceKey; + if (sourceTable == null || sourceKey == null) { + throw new Error('Expected parameter compaction row source columns'); + } + + return { + id: BigInt(row.id), + sourceTable, + sourceKey: Buffer.from(sourceKey), + lookup: Buffer.from(row.lookup), + bucketParameters: row.bucket_parameters ?? row.bucketParameters + }; +} + +function parameterKey(row: ParameterCompactionRow): string { + return `${row.lookup.toString('base64')}/${row.sourceTable}/${row.sourceKey.toString('base64')}`; +} + +function normalizedBucketParameters(value: unknown): string { + if (typeof value != 'string') { + return JSON.stringify(value ?? null); + } + + try { + const parsed = JSON.parse(value); + return typeof parsed == 'string' ? parsed : JSON.stringify(parsed); + } catch { + return value; + } +} + +function sameBucketParameters(left: unknown, right: unknown): boolean { + return normalizedBucketParameters(left) == normalizedBucketParameters(right); +} + +function isEmptyBucketParameters(value: unknown): boolean { + return normalizedBucketParameters(value) == '[]'; +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzlePersistedBatch.ts b/modules/module-drizzle-storage/src/storage/DrizzlePersistedBatch.ts new file mode 100644 index 000000000..2a23da15a --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzlePersistedBatch.ts @@ -0,0 +1,421 @@ +import type { Logger } from '@powersync/lib-services-framework'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import { inArray } from 'drizzle-orm'; +import * as uuid from 'uuid'; +import { bucketData, bucketParameters, type CurrentDataRow } from '../drivers/sqlite/schema.js'; +import type { DrizzleStorageTransaction } from '../drivers/sqlite/sqlite-config.js'; +import type { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +export interface DrizzlePersistedBatchOptions { + tx: DrizzleStorageTransaction; + dialect: DrizzleStorageDialect; + logger: Logger; + syncRules: sync_rules.HydratedSyncConfig; + replicationStreamId: number; + storeCurrentData: boolean; + skipExistingRows: boolean; + markRecordUnavailable: storage.BucketStorageMarkRecordUnavailable | undefined; +} + +export type CurrentBucket = { + bucket: string; + table: string; + id: string; +}; + +const MAX_ROW_SIZE = 15 * 1024 * 1024; + +/** + * Handles the writes for a single persisted operation chunk. + * + * `DrizzleBucketBatch` owns the public batch lifecycle, checkpointing, and + * listener hooks. This class owns the transactional write state for bucket + * data, parameter rows, and current data so large flushes can be split into + * smaller persisted chunks without keeping every tracked entity in one unit of + * work. + */ +export class DrizzlePersistedBatch { + private readonly currentDataById = new Map(); + + constructor(private readonly options: DrizzlePersistedBatchOptions) {} + + persistOperations(operations: storage.SaveOptions[], nextOpId: bigint): bigint { + this.loadCurrentData(operations); + for (const operation of operations) { + nextOpId = this.persistOperation(operation, nextOpId); + } + return nextOpId; + } + + persistBucketData(options: { + table: storage.SourceTable; + sourceKey: storage.ReplicaId; + existingBuckets: CurrentBucket[]; + evaluated: sync_rules.EvaluatedRow[]; + nextOpId: bigint; + }): bigint { + const remainingBuckets = new Map(options.existingBuckets.map((bucket) => [currentBucketKey(bucket), bucket])); + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + const deleteChecksum = utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)); + let nextOpId = options.nextOpId; + const values: (typeof bucketData.$inferInsert)[] = []; + + for (const row of options.evaluated) { + remainingBuckets.delete(currentBucketKey(row)); + const data = JSONBig.stringify(row.data); + const checksum = utils.hashData(row.table, row.id, data); + values.push({ + id: uuid.v4(), + groupId: this.options.replicationStreamId, + bucketName: row.bucket, + opId: nextOpId++, + op: 'PUT', + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + tableName: row.table, + rowId: row.id, + checksum: BigInt(checksum), + data, + targetOp: null + }); + } + + for (const bucket of remainingBuckets.values()) { + values.push({ + id: uuid.v4(), + groupId: this.options.replicationStreamId, + bucketName: bucket.bucket, + opId: nextOpId++, + op: 'REMOVE', + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + tableName: bucket.table, + rowId: bucket.id, + checksum: BigInt(deleteChecksum), + data: null, + targetOp: null + }); + } + + if (values.length > 0) { + this.options.tx.insert(this.options.dialect.tables.bucketData).values(values).run(); + } + + return nextOpId; + } + + persistParameterData(options: { + table: storage.SourceTable; + sourceKey: storage.ReplicaId; + existingLookups: Buffer[]; + evaluated: sync_rules.EvaluatedParameters[]; + nextOpId: bigint; + }): bigint { + const remainingLookups = new Map(options.existingLookups.map((lookup) => [lookup.toString('base64'), lookup])); + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + let nextOpId = options.nextOpId; + const values: (typeof bucketParameters.$inferInsert)[] = []; + + for (const row of options.evaluated) { + const lookup = storage.serializeLookupBuffer(row.lookup); + remainingLookups.delete(lookup.toString('base64')); + values.push({ + id: nextOpId++, + groupId: this.options.replicationStreamId, + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + lookup, + bucketParameters: JSONBig.stringify(row.bucketParameters) + }); + } + + for (const lookup of remainingLookups.values()) { + values.push({ + id: nextOpId++, + groupId: this.options.replicationStreamId, + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + lookup, + bucketParameters: '[]' + }); + } + + if (values.length > 0) { + this.options.tx.insert(this.options.dialect.tables.bucketParameters).values(values).run(); + } + + return nextOpId; + } + + private persistOperation(record: storage.SaveOptions, nextOpId: bigint): bigint { + const sourceTable = record.sourceTable; + const tableId = String(sourceTable.id); + const afterId = record.afterReplicaId ?? null; + const beforeId = record.beforeReplicaId ?? record.afterReplicaId; + const serializedBeforeId = storage.serializeReplicaId(beforeId); + const existingCurrentDataId = currentDataId(this.options.replicationStreamId, tableId, serializedBeforeId); + const existingCurrentData = this.currentDataById.get(existingCurrentDataId) ?? null; + + const storeCurrentData = this.options.storeCurrentData && sourceTable.storeCurrentData; + let existingBuckets = currentBuckets(existingCurrentData); + let existingLookups = currentLookups(existingCurrentData); + let after: sync_rules.ToastableSqliteRow | null | undefined = record.after; + + if (this.options.skipExistingRows) { + if (record.tag == storage.SaveOperationTag.INSERT) { + if (existingCurrentData != null) { + return nextOpId; + } + } else { + throw new ReplicationAssertionError(`${record.tag} not supported with skipExistingRows: true`); + } + } + + if (record.tag == storage.SaveOperationTag.UPDATE) { + if (existingCurrentData != null && storeCurrentData) { + after = storage.mergeToast(record.after, storage.deserializeBson(Buffer.from(existingCurrentData.data))); + } else if (existingCurrentData == null && storeCurrentData) { + this.options.markRecordUnavailable?.(record); + } + } + + if (beforeId != null && (afterId == null || !storage.replicaIdEquals(beforeId, afterId))) { + if (sourceTable.syncData) { + nextOpId = this.persistBucketData({ + table: sourceTable, + sourceKey: beforeId, + existingBuckets, + evaluated: [], + nextOpId + }); + existingBuckets = []; + } + + if (sourceTable.syncParameters) { + nextOpId = this.persistParameterData({ + table: sourceTable, + sourceKey: beforeId, + existingLookups, + evaluated: [], + nextOpId + }); + existingLookups = []; + } + } + + let newBuckets: CurrentBucket[] = []; + let newLookups: Buffer[] = []; + let afterData: Buffer | undefined; + let afterDataWasTruncated = false; + if (afterId != null && after != null && utils.isCompleteRow(storeCurrentData, after)) { + if (storeCurrentData) { + const prepared = this.serializeCurrentData(record, after); + after = prepared.after; + afterData = prepared.data; + afterDataWasTruncated = prepared.truncated; + } else { + afterData = storage.serializeBson({}); + } + + if (sourceTable.syncData) { + const { results: rawResults, errors } = this.options.syncRules.evaluateRowWithErrors({ + record: after as sync_rules.SqliteRow, + sourceTable: sourceTable.ref, + bucketDataSources: sourceTable.bucketDataSources + }); + const results = afterDataWasTruncated ? rawResults.filter(hasUsableObjectId) : rawResults; + for (const error of errors) { + this.options.logger.error( + `Failed to evaluate data query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + nextOpId = this.persistBucketData({ + table: sourceTable, + sourceKey: afterId, + existingBuckets, + evaluated: results, + nextOpId + }); + newBuckets = results.map((row) => ({ + bucket: row.bucket, + table: row.table, + id: row.id + })); + } + + if (sourceTable.syncParameters) { + const { results, errors } = this.options.syncRules.evaluateParameterRowWithErrors( + sourceTable.ref, + after as sync_rules.SqliteRow, + { + parameterLookupSources: sourceTable.parameterLookupSources + } + ); + for (const error of errors) { + this.options.logger.error( + `Failed to evaluate parameter query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + nextOpId = this.persistParameterData({ + table: sourceTable, + sourceKey: afterId, + existingLookups, + evaluated: results, + nextOpId + }); + newLookups = results.map((row) => storage.serializeLookupBuffer(row.lookup)); + } + } + + if (afterId != null && afterData != null) { + this.upsertCurrentData({ + tableId, + sourceKey: afterId, + buckets: newBuckets, + lookups: newLookups, + data: afterData, + pendingDelete: null + }); + } + + if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { + nextOpId = this.deleteCurrentData(tableId, beforeId, nextOpId); + } + + return nextOpId; + } + + private loadCurrentData(operations: storage.SaveOptions[]): void { + const ids = new Set(); + for (const operation of operations) { + const tableId = String(operation.sourceTable.id); + const beforeId = operation.beforeReplicaId ?? operation.afterReplicaId; + ids.add(currentDataId(this.options.replicationStreamId, tableId, storage.serializeReplicaId(beforeId))); + + const afterId = operation.afterReplicaId ?? null; + if (afterId != null) { + ids.add(currentDataId(this.options.replicationStreamId, tableId, storage.serializeReplicaId(afterId))); + } + } + + const rows = + ids.size == 0 + ? [] + : this.options.tx + .select() + .from(this.options.dialect.tables.currentData) + .where(inArray(this.options.dialect.tables.currentData.id, [...ids])) + .all(); + for (const row of rows) { + this.currentDataById.set(row.id, row); + } + } + + private serializeCurrentData( + record: storage.SaveOptions, + after: sync_rules.ToastableSqliteRow + ): { after: sync_rules.ToastableSqliteRow; data: Buffer; truncated: boolean } { + try { + const serialized = storage.serializeBson(after); + if (serialized.byteLength > MAX_ROW_SIZE) { + throw new Error(`Row too large: ${serialized.byteLength}`); + } + return { after, data: serialized, truncated: false }; + } catch (e) { + const error = e instanceof Error ? e : new Error(String(e)); + this.options.logger.warn( + `Data too big on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.message}` + ); + + // Keep the current_data row present, but drop field values. This mirrors + // the Postgres storage behavior for oversized BSON payloads and allows + // future TOAST-style updates to be marked unavailable instead of crashing + // the replication batch. + const emptyValues = Object.fromEntries(Object.keys(after).map((key) => [key, undefined])); + return { after: emptyValues, data: storage.serializeBson(emptyValues), truncated: true }; + } + } + + private upsertCurrentData(options: { + tableId: string; + sourceKey: storage.ReplicaId; + buckets: CurrentBucket[]; + lookups: Buffer[]; + data: Buffer; + pendingDelete: bigint | null; + }): void { + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + const id = currentDataId(this.options.replicationStreamId, options.tableId, serializedSourceKey); + const payload = { + id, + groupId: this.options.replicationStreamId, + sourceTable: options.tableId, + sourceKey: serializedSourceKey, + buckets: options.buckets, + lookups: options.lookups.map((lookup) => lookup.toString('hex')), + data: options.data, + pendingDelete: options.pendingDelete + }; + this.options.tx + .insert(this.options.dialect.tables.currentData) + .values(payload) + .onConflictDoUpdate({ + target: this.options.dialect.tables.currentData.id, + set: payload + }) + .run(); + this.currentDataById.set(id, payload); + } + + private deleteCurrentData(tableId: string, sourceKey: storage.ReplicaId, nextOpId: bigint): bigint { + const serializedSourceKey = storage.serializeReplicaId(sourceKey); + const id = currentDataId(this.options.replicationStreamId, tableId, serializedSourceKey); + const payload = { + id, + groupId: this.options.replicationStreamId, + sourceTable: tableId, + sourceKey: serializedSourceKey, + buckets: [], + lookups: [], + data: storage.serializeBson({}), + pendingDelete: nextOpId + }; + this.options.tx + .insert(this.options.dialect.tables.currentData) + .values(payload) + .onConflictDoUpdate({ target: this.options.dialect.tables.currentData.id, set: payload }) + .run(); + this.currentDataById.set(id, payload); + return nextOpId + 1n; + } +} + +export function currentBuckets(row: CurrentDataRow | null): CurrentBucket[] { + return Array.isArray(row?.buckets) ? (row.buckets as CurrentBucket[]) : []; +} + +export function currentLookups(row: CurrentDataRow | null): Buffer[] { + return Array.isArray(row?.lookups) ? (row.lookups as string[]).map((lookup) => Buffer.from(lookup, 'hex')) : []; +} + +function currentDataId(groupId: number, sourceTable: string, sourceKey: Buffer): string { + return `${groupId}:${sourceTable}:${sourceKey.toString('hex')}`; +} + +function currentBucketKey(bucket: CurrentBucket | sync_rules.EvaluatedRow): string { + return `${bucket.bucket}/${bucket.table}/${bucket.id}`; +} + +function hasUsableObjectId(row: sync_rules.EvaluatedRow): boolean { + return row.id !== '' || row.data.id != null; +} + +function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string { + if (storage.isUUID(id)) { + return `${tableId}/${id.toHexString()}`; + } + return uuid.v5(storage.serializeBson({ table: tableId, id }), utils.ID_NAMESPACE); +} diff --git a/modules/module-drizzle-storage/test/src/__snapshots__/storage.test.ts.snap b/modules/module-drizzle-storage/test/src/__snapshots__/storage.test.ts.snap new file mode 100644 index 000000000..5744533d0 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/__snapshots__/storage.test.ts.snap @@ -0,0 +1,124 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v1 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v1 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v2 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v2 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v3 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v3 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; diff --git a/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap new file mode 100644 index 000000000..db64ec935 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -0,0 +1,3568 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`sync - Drizzle SQLite > storage v1 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v2 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v3 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; diff --git a/modules/module-drizzle-storage/test/src/column-types.test.ts b/modules/module-drizzle-storage/test/src/column-types.test.ts new file mode 100644 index 000000000..625cc8359 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/column-types.test.ts @@ -0,0 +1,47 @@ +import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import { describe, expect, it } from 'vitest'; +import { + createSqliteDrizzleRuntime, + DRIZZLE_SQLITE_STORAGE_TYPE, + normalizeDrizzleSqliteStorageConfig, + SQLITE_DRIZZLE_MIGRATIONS_PATH, + sqliteSchema +} from '../../src/index.js'; + +describe('Drizzle SQLite column mappings', () => { + it('round-trips runtime values without bigint precision loss', () => { + const runtime = createSqliteDrizzleRuntime( + normalizeDrizzleSqliteStorageConfig({ + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: ':memory:' + }) + ); + try { + migrate(runtime.db, { migrationsFolder: SQLITE_DRIZZLE_MIGRATIONS_PATH }); + const opId = BigInt(Number.MAX_SAFE_INTEGER) + 100n; + runtime.db + .insert(sqliteSchema.bucketData) + .values({ + id: '1', + groupId: 1, + bucketName: 'bucket', + opId, + op: 'PUT', + checksum: 2n, + sourceTable: 'table', + sourceKey: Buffer.from('key'), + tableName: 'items', + rowId: 'row', + data: '{}', + targetOp: null + }) + .run(); + const row = runtime.db.select().from(sqliteSchema.bucketData).get()!; + expect(row.opId).toBe(opId); + expect(row.checksum).toBe(2n); + expect(row.sourceKey).toEqual(Buffer.from('key')); + } finally { + runtime.close(); + } + }); +}); diff --git a/modules/module-drizzle-storage/test/src/storage-provider.test.ts b/modules/module-drizzle-storage/test/src/storage-provider.test.ts new file mode 100644 index 000000000..0e03335c7 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/storage-provider.test.ts @@ -0,0 +1,34 @@ +import { storage, system } from '@powersync/service-core'; +import { describe, expect, it } from 'vitest'; +import { DRIZZLE_SQLITE_STORAGE_TYPE, DrizzleStorageProvider } from '../../src/index.js'; + +const RESOLVED_CONFIG = { + storage: { type: DRIZZLE_SQLITE_STORAGE_TYPE, filename: ':memory:' }, + slot_name_prefix: 'test_' +} as const; + +function getStorageOptions(serviceMode: system.ServiceContextMode): storage.GetStorageOptions { + return { + resolvedConfig: RESOLVED_CONFIG as unknown as storage.GetStorageOptions['resolvedConfig'], + serviceMode + }; +} + +describe('Drizzle SQLite storage provider', () => { + it('rejects split service runners', async () => { + const provider = new DrizzleStorageProvider(); + await expect(provider.getStorage(getStorageOptions(system.ServiceContextMode.API))).rejects.toThrow( + 'Drizzle SQLite storage only supports the unified service runner' + ); + await expect(provider.getStorage(getStorageOptions(system.ServiceContextMode.SYNC))).rejects.toThrow( + 'Drizzle SQLite storage only supports the unified service runner' + ); + }); + + it('allows the unified service runner', async () => { + const activeStorage = await new DrizzleStorageProvider().getStorage( + getStorageOptions(system.ServiceContextMode.UNIFIED) + ); + await activeStorage.shutDown(); + }); +}); diff --git a/modules/module-drizzle-storage/test/src/storage.test.ts b/modules/module-drizzle-storage/test/src/storage.test.ts new file mode 100644 index 000000000..93ac9c9cb --- /dev/null +++ b/modules/module-drizzle-storage/test/src/storage.test.ts @@ -0,0 +1,21 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { DRIZZLE_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe('Sync Bucket Validation', register.registerBucketValidationTests); + +for (let storageVersion of TEST_STORAGE_VERSIONS) { + describe(`Drizzle SQLite Sync Bucket Storage - Parameters - v${storageVersion}`, () => + register.registerDataStorageParameterTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`Drizzle SQLite Sync Bucket Storage - Data - v${storageVersion}`, () => + register.registerDataStorageDataTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`Drizzle SQLite Sync Bucket Storage - Checkpoints - v${storageVersion}`, () => + register.registerDataStorageCheckpointTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`Drizzle SQLite Sync Bucket Storage - Compaction - v${storageVersion}`, () => { + register.registerCompactTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion }); + register.registerParameterCompactTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion }); + }); +} diff --git a/modules/module-drizzle-storage/test/src/storage_bench.test.ts b/modules/module-drizzle-storage/test/src/storage_bench.test.ts new file mode 100644 index 000000000..e2f82e334 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/storage_bench.test.ts @@ -0,0 +1,18 @@ +import type { StorageBenchmarkResult } from '@powersync/service-core-tests'; +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { DRIZZLE_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +const results: StorageBenchmarkResult[] = []; +register.registerStorageBenchmarkSummary(results); + +describe.sequential('Drizzle SQLite Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion }, + { storageName: 'drizzle:sqlite', storageVersion, results } + ); + }); + } +}); diff --git a/modules/module-drizzle-storage/test/src/storage_sync.test.ts b/modules/module-drizzle-storage/test/src/storage_sync.test.ts new file mode 100644 index 000000000..408707c0f --- /dev/null +++ b/modules/module-drizzle-storage/test/src/storage_sync.test.ts @@ -0,0 +1,14 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { DRIZZLE_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe('sync - Drizzle SQLite', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`storage v${storageVersion}`, () => { + register.registerSyncTests(DRIZZLE_SQLITE_STORAGE_FACTORY.factory, { + storageVersion, + tableIdStrings: DRIZZLE_SQLITE_STORAGE_FACTORY.tableIdStrings + }); + }); + } +}); diff --git a/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts b/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts new file mode 100644 index 000000000..985e225df --- /dev/null +++ b/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts @@ -0,0 +1,175 @@ +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { eq } from 'drizzle-orm'; +import { describe, expect, it } from 'vitest'; +import type { DrizzleBucketStorageFactory } from '../../src/index.js'; +import { DRIZZLE_SQLITE_STORAGE_FACTORY } from './util.js'; + +describe('Drizzle SyncRules storage', () => { + const syncRules = updateSyncRulesFromYaml( + ` +bucket_definitions: + mybucket: + data: [] +`, + { + validate: false + } + ); + + it('stores and resolves managed write checkpoints', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + + const first = await bucketStorage.createManagedWriteCheckpoint({ + user_id: 'user1', + heads: { '1': '5/0' } + }); + const second = await bucketStorage.createManagedWriteCheckpoint({ + user_id: 'user1', + heads: { '1': '6/0' } + }); + + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '4/0' } })).resolves.toBeNull(); + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '5/0' } })).resolves.toBe(first); + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '6/0' } })).resolves.toBe(second); + }); + + it('watches checkpoint and managed write checkpoint changes', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + const abortController = new AbortController(); + + try { + const iterator = bucketStorage + .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) + [Symbol.asyncIterator](); + + const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({ + user_id: 'user1', + heads: { '1': '5/0' } + }); + + factory.dialect.db + .update(factory.dialect.tables.syncRules) + .set({ + lastCheckpoint: 0n, + lastCheckpointLsn: '5/0' + }) + .where(eq(factory.dialect.tables.syncRules.id, stream.replicationStreamId)) + .run(); + factory.checkpointWatcher.notify(); + + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { + base: { + checkpoint: 0n, + lsn: '5/0' + }, + writeCheckpoint + } + }); + } finally { + abortController.abort(); + } + }); + + it('resolves custom write checkpoints from the write checkpoint entity', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); + + factory.dialect.db + .insert(factory.dialect.tables.writeCheckpoints) + .values({ + id: 'custom-user1', + syncRulesId: stream.replicationStreamId, + userId: 'user1', + checkpoint: 42n, + heads: null, + createdAt: new Date() + }) + .run(); + + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1' })).resolves.toBe(42n); + }); + + it('marks newly resolved source tables as requiring an initial snapshot', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +bucket_definitions: + global: + data: + - SELECT * FROM lists +`, + { + validate: false + } + ) + ); + const bucketStorage = factory.getInstance(stream); + await using writer = await bucketStorage.createWriter({ + defaultSchema: 'public', + zeroLSN: '0/0', + storeCurrentData: true + }); + + const resolved = await writer.resolveTables({ + connection_id: 1, + source: { + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: 123, + schema: 'public', + name: 'lists', + replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] + }, + idGenerator: () => 'lists-table' + }); + + expect(resolved.tables[0]?.snapshotComplete).toBe(false); + + await writer.markTableSnapshotDone(resolved.tables, '0/1'); + const resolvedAgain = await writer.resolveTables({ + connection_id: 1, + source: { + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: 123, + schema: 'public', + name: 'lists', + replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] + } + }); + + expect(resolvedAgain.tables[0]?.snapshotComplete).toBe(true); + }); + + it('returns active and processing streams as replicating streams', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId, + state: storage.SyncRuleState.PROCESSING + } + ]); + + factory.dialect.db + .update(factory.dialect.tables.syncRules) + .set({ state: storage.SyncRuleState.ACTIVE }) + .where(eq(factory.dialect.tables.syncRules.id, stream.replicationStreamId)) + .run(); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId, + state: storage.SyncRuleState.ACTIVE + } + ]); + }); +}); diff --git a/modules/module-drizzle-storage/test/src/util.ts b/modules/module-drizzle-storage/test/src/util.ts new file mode 100644 index 000000000..bd874b4f9 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/util.ts @@ -0,0 +1,27 @@ +import { storage, SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core'; +import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import { + createSqliteDrizzleStorageFactory, + DRIZZLE_SQLITE_STORAGE_TYPE, + normalizeDrizzleSqliteStorageConfig, + SQLITE_DRIZZLE_MIGRATIONS_PATH +} from '../../src/index.js'; + +const BASE_CONFIG = { + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: ':memory:' +} as const; + +export const DRIZZLE_SQLITE_STORAGE_FACTORY: storage.TestStorageConfig = { + tableIdStrings: true, + factory: async () => { + const factory = createSqliteDrizzleStorageFactory({ + config: normalizeDrizzleSqliteStorageConfig(BASE_CONFIG), + slotNamePrefix: 'test_' + }); + migrate(factory.runtime.db, { migrationsFolder: SQLITE_DRIZZLE_MIGRATIONS_PATH }); + return factory; + } +}; + +export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index e6a6d98ce..78a6921d4 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -47,6 +47,7 @@ "@powersync/lib-service-postgres": "workspace:*", "@powersync/service-core-tests": "workspace:*", "@powersync/service-module-mikroorm-storage": "workspace:*", + "@powersync/service-module-drizzle-storage": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-postgres-storage": "workspace:*", "@types/semver": "^7.5.4" diff --git a/modules/module-postgres/test/src/env.ts b/modules/module-postgres/test/src/env.ts index d3b80f2c1..77b62cbd5 100644 --- a/modules/module-postgres/test/src/env.ts +++ b/modules/module-postgres/test/src/env.ts @@ -11,5 +11,7 @@ export const env = utils.collectEnvironmentVariables({ TEST_MONGO_STORAGE: utils.type.boolean.default('false'), TEST_POSTGRES_STORAGE: utils.type.boolean.default('false'), TEST_MIKROORM_SQLITE_STORAGE: utils.type.boolean.default('true'), - MIKROORM_SQLITE_STORAGE_TEST_FILENAME: utils.type.string.default('') + MIKROORM_SQLITE_STORAGE_TEST_FILENAME: utils.type.string.default(''), + TEST_DRIZZLE_SQLITE_STORAGE: utils.type.boolean.default('false'), + DRIZZLE_SQLITE_STORAGE_TEST_FILENAME: utils.type.string.default('') }); diff --git a/modules/module-postgres/test/src/util.ts b/modules/module-postgres/test/src/util.ts index d20195ed2..03ca2d60c 100644 --- a/modules/module-postgres/test/src/util.ts +++ b/modules/module-postgres/test/src/util.ts @@ -10,13 +10,14 @@ import { TestStorageFactory } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; +import * as drizzle_storage from '@powersync/service-module-drizzle-storage'; import * as mikroorm_storage from '@powersync/service-module-mikroorm-storage'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as postgres_storage from '@powersync/service-module-postgres-storage'; import { existsSync } from 'node:fs'; import { unlink } from 'node:fs/promises'; -import { join } from 'node:path'; import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, TestOptions } from 'vitest'; import { env } from './env.js'; @@ -39,6 +40,13 @@ const DEFAULT_MIKROORM_SQLITE_STORAGE_FILENAME = join( const MIKROORM_SQLITE_STORAGE_FILENAME = env.MIKROORM_SQLITE_STORAGE_TEST_FILENAME || DEFAULT_MIKROORM_SQLITE_STORAGE_FILENAME; +const DEFAULT_DRIZZLE_SQLITE_STORAGE_FILENAME = join( + tmpdir(), + `powersync-postgres-replication-drizzle-storage-${process.pid}-${process.env.VITEST_WORKER_ID ?? '0'}.sqlite` +); +const DRIZZLE_SQLITE_STORAGE_FILENAME = + env.DRIZZLE_SQLITE_STORAGE_TEST_FILENAME || DEFAULT_DRIZZLE_SQLITE_STORAGE_FILENAME; + export const INITIALIZED_MIKROORM_SQLITE_STORAGE_FACTORY: TestStorageConfig = { tableIdStrings: true, factory: async (options) => { @@ -59,6 +67,24 @@ export const INITIALIZED_MIKROORM_SQLITE_STORAGE_FACTORY: TestStorageConfig = { } }; +export const INITIALIZED_DRIZZLE_SQLITE_STORAGE_FACTORY: TestStorageConfig = { + tableIdStrings: true, + factory: async (options) => { + if (!options?.doNotClear && existsSync(DRIZZLE_SQLITE_STORAGE_FILENAME)) { + await unlink(DRIZZLE_SQLITE_STORAGE_FILENAME); + } + const factory = drizzle_storage.createSqliteDrizzleStorageFactory({ + config: drizzle_storage.normalizeDrizzleSqliteStorageConfig({ + type: drizzle_storage.DRIZZLE_SQLITE_STORAGE_TYPE, + filename: DRIZZLE_SQLITE_STORAGE_FILENAME + }), + slotNamePrefix: 'test_' + }); + drizzle_storage.runSqliteDrizzleMigrations(factory.runtime); + return factory; + } +}; + const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; export interface StorageVersionTestContext { @@ -95,6 +121,10 @@ export function describeWithStorage( if (env.TEST_MIKROORM_SQLITE_STORAGE) { describeFactory('mikroorm sqlite', INITIALIZED_MIKROORM_SQLITE_STORAGE_FACTORY); } + + if (env.TEST_DRIZZLE_SQLITE_STORAGE) { + describeFactory('drizzle sqlite', INITIALIZED_DRIZZLE_SQLITE_STORAGE_FACTORY); + } } export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({ diff --git a/modules/module-postgres/test/tsconfig.json b/modules/module-postgres/test/tsconfig.json index cf5b32e4c..df3b2c86e 100644 --- a/modules/module-postgres/test/tsconfig.json +++ b/modules/module-postgres/test/tsconfig.json @@ -22,6 +22,9 @@ { "path": "../../module-mikroorm-storage" }, + { + "path": "../../module-drizzle-storage" + }, { "path": "../../module-postgres-storage" } diff --git a/packages/schema/src/scripts/compile-json-schema.ts b/packages/schema/src/scripts/compile-json-schema.ts index 5f584a049..5e70a7481 100644 --- a/packages/schema/src/scripts/compile-json-schema.ts +++ b/packages/schema/src/scripts/compile-json-schema.ts @@ -1,4 +1,5 @@ import { ConvexConnectionConfig } from '@powersync/service-module-convex/types'; +import { DrizzleStorageConfig } from '@powersync/service-module-drizzle-storage/types'; import { MikroOrmStorageConfig } from '@powersync/service-module-mikroorm-storage/types'; import { MongoStorageConfig } from '@powersync/service-module-mongodb-storage/types'; import { MongoConnectionConfig } from '@powersync/service-module-mongodb/types'; @@ -28,7 +29,8 @@ const mergedDataSourceConfig = configFile.genericDataSourceConfig const mergedStorageConfig = configFile.GenericStorageConfig.or(PostgresStorageConfig) .or(MongoStorageConfig) - .or(MikroOrmStorageConfig); + .or(MikroOrmStorageConfig) + .or(DrizzleStorageConfig); const mergedConfig = t.object({ ...baseShape, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca53802be..5e12d5616 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -94,10 +94,10 @@ importers: version: 6.0.3 vite: specifier: ^8.0.9 - version: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) + version: 8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) ws: specifier: ^8.2.3 version: 8.18.0 @@ -194,7 +194,7 @@ importers: version: 4.17.6 vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) modules/module-convex: dependencies: @@ -263,6 +263,49 @@ importers: specifier: ^5.8.5 version: 5.8.5 + modules/module-drizzle-storage: + dependencies: + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../../libs/lib-services + '@powersync/service-core': + specifier: workspace:* + version: link:../../packages/service-core + '@powersync/service-jsonbig': + specifier: workspace:* + version: link:../../packages/jsonbig + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + better-sqlite3: + specifier: ^12.10.0 + version: 12.10.0 + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.15.4)(better-sqlite3@12.10.0)(kysely@0.29.2)(mysql2@3.22.4(@types/node@25.5.0)) + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + uuid: + specifier: 'catalog:' + version: 14.0.0 + devDependencies: + '@powersync/service-core-tests': + specifier: workspace:* + version: link:../../packages/service-core-tests + '@types/better-sqlite3': + specifier: ^7.6.13 + version: 7.6.13 + drizzle-kit: + specifier: ^0.31.10 + version: 0.31.10 + typescript: + specifier: 'catalog:' + version: 6.0.3 + modules/module-mikroorm-storage: dependencies: '@mikro-orm/core': @@ -541,6 +584,9 @@ importers: '@powersync/service-core-tests': specifier: workspace:* version: link:../../packages/service-core-tests + '@powersync/service-module-drizzle-storage': + specifier: workspace:* + version: link:../module-drizzle-storage '@powersync/service-module-mikroorm-storage': specifier: workspace:* version: link:../module-mikroorm-storage @@ -614,7 +660,7 @@ importers: devDependencies: vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) packages/jsonbig: dependencies: @@ -655,6 +701,9 @@ importers: '@powersync/service-module-convex': specifier: workspace:* version: link:../../modules/module-convex + '@powersync/service-module-drizzle-storage': + specifier: workspace:* + version: link:../../modules/module-drizzle-storage '@powersync/service-module-mikroorm-storage': specifier: workspace:* version: link:../../modules/module-mikroorm-storage @@ -811,7 +860,7 @@ importers: version: link:../sync-rules vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) devDependencies: '@opentelemetry/sdk-metrics': specifier: ^1.30.1 @@ -854,7 +903,7 @@ importers: version: 1.0.0 vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) packages/types: dependencies: @@ -882,6 +931,9 @@ importers: '@powersync/service-module-core': specifier: workspace:* version: link:../modules/module-core + '@powersync/service-module-drizzle-storage': + specifier: workspace:* + version: link:../modules/module-drizzle-storage '@powersync/service-module-mikroorm-storage': specifier: workspace:* version: link:../modules/module-mikroorm-storage @@ -1147,6 +1199,9 @@ packages: '@dabh/diagnostics@2.0.3': resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} + '@drizzle-team/brocli@0.10.2': + resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} + '@emnapi/core@1.8.1': resolution: {integrity: sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==} @@ -1165,148 +1220,576 @@ packages: '@emnapi/wasi-threads@1.2.1': resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} + '@esbuild-kit/core-utils@3.3.2': + resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} + deprecated: 'Merged into tsx: https://tsx.hirok.io' + + '@esbuild-kit/esm-loader@2.6.5': + resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} + deprecated: 'Merged into tsx: https://tsx.hirok.io' + + '@esbuild/aix-ppc64@0.25.12': + resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/aix-ppc64@0.27.0': resolution: {integrity: sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.18.20': + resolution: {integrity: sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.25.12': + resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm64@0.27.0': resolution: {integrity: sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==} engines: {node: '>=18'} cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.18.20': + resolution: {integrity: sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.25.12': + resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-arm@0.27.0': resolution: {integrity: sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==} engines: {node: '>=18'} cpu: [arm] os: [android] + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.18.20': + resolution: {integrity: sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.25.12': + resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/android-x64@0.27.0': resolution: {integrity: sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==} engines: {node: '>=18'} cpu: [x64] os: [android] + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.18.20': + resolution: {integrity: sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.25.12': + resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-arm64@0.27.0': resolution: {integrity: sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.18.20': + resolution: {integrity: sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.12': + resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/darwin-x64@0.27.0': resolution: {integrity: sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==} engines: {node: '>=18'} cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.18.20': + resolution: {integrity: sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.25.12': + resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-arm64@0.27.0': resolution: {integrity: sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.18.20': + resolution: {integrity: sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.12': + resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/freebsd-x64@0.27.0': resolution: {integrity: sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.18.20': + resolution: {integrity: sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.25.12': + resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm64@0.27.0': resolution: {integrity: sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==} engines: {node: '>=18'} cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.18.20': + resolution: {integrity: sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.25.12': + resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-arm@0.27.0': resolution: {integrity: sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==} engines: {node: '>=18'} cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.18.20': + resolution: {integrity: sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.25.12': + resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-ia32@0.27.0': resolution: {integrity: sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==} engines: {node: '>=18'} cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.18.20': + resolution: {integrity: sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.25.12': + resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-loong64@0.27.0': resolution: {integrity: sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==} engines: {node: '>=18'} cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.18.20': + resolution: {integrity: sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.25.12': + resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-mips64el@0.27.0': resolution: {integrity: sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.18.20': + resolution: {integrity: sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.25.12': + resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-ppc64@0.27.0': resolution: {integrity: sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.18.20': + resolution: {integrity: sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.12': + resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-riscv64@0.27.0': resolution: {integrity: sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.18.20': + resolution: {integrity: sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.25.12': + resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-s390x@0.27.0': resolution: {integrity: sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==} engines: {node: '>=18'} cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.18.20': + resolution: {integrity: sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.25.12': + resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/linux-x64@0.27.0': resolution: {integrity: sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==} engines: {node: '>=18'} cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.12': + resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + '@esbuild/netbsd-arm64@0.27.0': resolution: {integrity: sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.18.20': + resolution: {integrity: sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.12': + resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + '@esbuild/netbsd-x64@0.27.0': resolution: {integrity: sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.12': + resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-arm64@0.27.0': resolution: {integrity: sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.18.20': + resolution: {integrity: sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.12': + resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/openbsd-x64@0.27.0': resolution: {integrity: sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.25.12': + resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + '@esbuild/openharmony-arm64@0.27.0': resolution: {integrity: sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.18.20': + resolution: {integrity: sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.25.12': + resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/sunos-x64@0.27.0': resolution: {integrity: sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==} engines: {node: '>=18'} cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.18.20': + resolution: {integrity: sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.25.12': + resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-arm64@0.27.0': resolution: {integrity: sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==} engines: {node: '>=18'} - cpu: [arm64] + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.18.20': + resolution: {integrity: sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.25.12': + resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} + engines: {node: '>=18'} + cpu: [ia32] os: [win32] '@esbuild/win32-ia32@0.27.0': @@ -1315,12 +1798,36 @@ packages: cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.18.20': + resolution: {integrity: sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.25.12': + resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@esbuild/win32-x64@0.27.0': resolution: {integrity: sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==} engines: {node: '>=18'} cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@fastify/ajv-compiler@4.0.5': resolution: {integrity: sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==} @@ -2140,6 +2647,9 @@ packages: '@types/async@3.2.24': resolution: {integrity: sha512-8iHVLHsCCOBKjCF2KwFe0p9Z3rfM9mL+sSP8btyR5vTjJRAqpBYD28/ZLgXPf0pjG1VxOvtCV/BgXkQbpSe8Hw==} + '@types/better-sqlite3@7.6.13': + resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} + '@types/body-parser@1.19.6': resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} @@ -2474,6 +2984,9 @@ packages: buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} @@ -2700,6 +3213,102 @@ packages: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} engines: {node: '>=12'} + drizzle-kit@0.31.10: + resolution: {integrity: sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw==} + hasBin: true + + drizzle-orm@0.45.2: + resolution: {integrity: sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q==} + peerDependencies: + '@aws-sdk/client-rds-data': '>=3' + '@cloudflare/workers-types': '>=4' + '@electric-sql/pglite': '>=0.2.0' + '@libsql/client': '>=0.10.0' + '@libsql/client-wasm': '>=0.10.0' + '@neondatabase/serverless': '>=0.10.0' + '@op-engineering/op-sqlite': '>=2' + '@opentelemetry/api': ^1.4.1 + '@planetscale/database': '>=1.13' + '@prisma/client': '*' + '@tidbcloud/serverless': '*' + '@types/better-sqlite3': '*' + '@types/pg': '*' + '@types/sql.js': '*' + '@upstash/redis': '>=1.34.7' + '@vercel/postgres': '>=0.8.0' + '@xata.io/client': '*' + better-sqlite3: '>=7' + bun-types: '*' + expo-sqlite: '>=14.0.0' + gel: '>=2' + knex: '*' + kysely: '*' + mysql2: '>=2' + pg: '>=8' + postgres: '>=3' + prisma: '*' + sql.js: '>=1' + sqlite3: '>=5' + peerDependenciesMeta: + '@aws-sdk/client-rds-data': + optional: true + '@cloudflare/workers-types': + optional: true + '@electric-sql/pglite': + optional: true + '@libsql/client': + optional: true + '@libsql/client-wasm': + optional: true + '@neondatabase/serverless': + optional: true + '@op-engineering/op-sqlite': + optional: true + '@opentelemetry/api': + optional: true + '@planetscale/database': + optional: true + '@prisma/client': + optional: true + '@tidbcloud/serverless': + optional: true + '@types/better-sqlite3': + optional: true + '@types/pg': + optional: true + '@types/sql.js': + optional: true + '@upstash/redis': + optional: true + '@vercel/postgres': + optional: true + '@xata.io/client': + optional: true + better-sqlite3: + optional: true + bun-types: + optional: true + expo-sqlite: + optional: true + gel: + optional: true + knex: + optional: true + kysely: + optional: true + mysql2: + optional: true + pg: + optional: true + postgres: + optional: true + prisma: + optional: true + sql.js: + optional: true + sqlite3: + optional: true + ebnf@1.9.1: resolution: {integrity: sha512-uW2UKSsuty9ANJ3YByIQE4ANkD8nqUPO7r6Fwcc1ADKPe9FRdcPpMl3VEput4JSvKBJ4J86npIC2MLP0pYkCuw==} hasBin: true @@ -2723,11 +3332,26 @@ packages: es-module-lexer@2.0.0: resolution: {integrity: sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==} + esbuild@0.18.20: + resolution: {integrity: sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==} + engines: {node: '>=12'} + hasBin: true + + esbuild@0.25.12: + resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} + engines: {node: '>=18'} + hasBin: true + esbuild@0.27.0: resolution: {integrity: sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==} engines: {node: '>=18'} hasBin: true + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + escalade@3.1.2: resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} engines: {node: '>=6'} @@ -2890,6 +3514,9 @@ packages: resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} engines: {node: '>=18'} + get-tsconfig@4.14.0: + resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} + github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} @@ -3814,6 +4441,9 @@ packages: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + resolve@1.22.8: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} hasBin: true @@ -3969,6 +4599,13 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + source-map-support@0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + sparse-bitfield@3.0.3: resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} @@ -4158,6 +4795,11 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.23.0: + resolution: {integrity: sha512-eUdUIaCr963q2h5u3+QwvYp0+eqPvn+egeqZUm0hwERCqqx1E3kK5ehbGCvqSE5MQAULr67ww0cA3jKc3YkM1w==} + engines: {node: '>=18.0.0'} + hasBin: true + tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} @@ -4774,6 +5416,8 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 + '@drizzle-team/brocli@0.10.2': {} + '@emnapi/core@1.8.1': dependencies: '@emnapi/wasi-threads': 1.1.0 @@ -4806,84 +5450,316 @@ snapshots: tslib: 2.8.1 optional: true + '@esbuild-kit/core-utils@3.3.2': + dependencies: + esbuild: 0.18.20 + source-map-support: 0.5.21 + + '@esbuild-kit/esm-loader@2.6.5': + dependencies: + '@esbuild-kit/core-utils': 3.3.2 + get-tsconfig: 4.14.0 + + '@esbuild/aix-ppc64@0.25.12': + optional: true + '@esbuild/aix-ppc64@0.27.0': optional: true + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.18.20': + optional: true + + '@esbuild/android-arm64@0.25.12': + optional: true + '@esbuild/android-arm64@0.27.0': optional: true + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.18.20': + optional: true + + '@esbuild/android-arm@0.25.12': + optional: true + '@esbuild/android-arm@0.27.0': optional: true + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.18.20': + optional: true + + '@esbuild/android-x64@0.25.12': + optional: true + '@esbuild/android-x64@0.27.0': optional: true + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.18.20': + optional: true + + '@esbuild/darwin-arm64@0.25.12': + optional: true + '@esbuild/darwin-arm64@0.27.0': optional: true + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.18.20': + optional: true + + '@esbuild/darwin-x64@0.25.12': + optional: true + '@esbuild/darwin-x64@0.27.0': optional: true + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.18.20': + optional: true + + '@esbuild/freebsd-arm64@0.25.12': + optional: true + '@esbuild/freebsd-arm64@0.27.0': optional: true + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.18.20': + optional: true + + '@esbuild/freebsd-x64@0.25.12': + optional: true + '@esbuild/freebsd-x64@0.27.0': optional: true + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.18.20': + optional: true + + '@esbuild/linux-arm64@0.25.12': + optional: true + '@esbuild/linux-arm64@0.27.0': optional: true + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.18.20': + optional: true + + '@esbuild/linux-arm@0.25.12': + optional: true + '@esbuild/linux-arm@0.27.0': optional: true + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.18.20': + optional: true + + '@esbuild/linux-ia32@0.25.12': + optional: true + '@esbuild/linux-ia32@0.27.0': optional: true + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.18.20': + optional: true + + '@esbuild/linux-loong64@0.25.12': + optional: true + '@esbuild/linux-loong64@0.27.0': optional: true + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.18.20': + optional: true + + '@esbuild/linux-mips64el@0.25.12': + optional: true + '@esbuild/linux-mips64el@0.27.0': optional: true + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.18.20': + optional: true + + '@esbuild/linux-ppc64@0.25.12': + optional: true + '@esbuild/linux-ppc64@0.27.0': optional: true + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.18.20': + optional: true + + '@esbuild/linux-riscv64@0.25.12': + optional: true + '@esbuild/linux-riscv64@0.27.0': optional: true + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.18.20': + optional: true + + '@esbuild/linux-s390x@0.25.12': + optional: true + '@esbuild/linux-s390x@0.27.0': optional: true + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.18.20': + optional: true + + '@esbuild/linux-x64@0.25.12': + optional: true + '@esbuild/linux-x64@0.27.0': optional: true + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.25.12': + optional: true + '@esbuild/netbsd-arm64@0.27.0': optional: true + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.18.20': + optional: true + + '@esbuild/netbsd-x64@0.25.12': + optional: true + '@esbuild/netbsd-x64@0.27.0': optional: true + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.25.12': + optional: true + '@esbuild/openbsd-arm64@0.27.0': optional: true + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.18.20': + optional: true + + '@esbuild/openbsd-x64@0.25.12': + optional: true + '@esbuild/openbsd-x64@0.27.0': optional: true + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.25.12': + optional: true + '@esbuild/openharmony-arm64@0.27.0': optional: true + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.18.20': + optional: true + + '@esbuild/sunos-x64@0.25.12': + optional: true + '@esbuild/sunos-x64@0.27.0': optional: true + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.18.20': + optional: true + + '@esbuild/win32-arm64@0.25.12': + optional: true + '@esbuild/win32-arm64@0.27.0': optional: true + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.18.20': + optional: true + + '@esbuild/win32-ia32@0.25.12': + optional: true + '@esbuild/win32-ia32@0.27.0': optional: true + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.18.20': + optional: true + + '@esbuild/win32-x64@0.25.12': + optional: true + '@esbuild/win32-x64@0.27.0': optional: true + '@esbuild/win32-x64@0.28.1': + optional: true + '@fastify/ajv-compiler@4.0.5': dependencies: ajv: 8.18.0 @@ -5759,6 +6635,10 @@ snapshots: '@types/async@3.2.24': {} + '@types/better-sqlite3@7.6.13': + dependencies: + '@types/node': 22.16.2 + '@types/body-parser@1.19.6': dependencies: '@types/connect': 3.4.38 @@ -5904,7 +6784,7 @@ snapshots: obug: 2.1.1 std-env: 4.0.0 tinyrainbow: 3.1.0 - vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) '@vitest/expect@4.1.5': dependencies: @@ -5915,21 +6795,21 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3))': + '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) - '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3))': + '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) '@vitest/pretty-format@4.1.5': dependencies: @@ -5958,7 +6838,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) '@vitest/utils@4.1.5': dependencies: @@ -6130,6 +7010,8 @@ snapshots: buffer-equal-constant-time@1.0.1: {} + buffer-from@1.1.2: {} + buffer@5.7.1: dependencies: base64-js: 1.5.1 @@ -6312,6 +7194,22 @@ snapshots: dotenv@16.4.5: {} + drizzle-kit@0.31.10: + dependencies: + '@drizzle-team/brocli': 0.10.2 + '@esbuild-kit/esm-loader': 2.6.5 + esbuild: 0.25.12 + tsx: 4.23.0 + + drizzle-orm@0.45.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.15.4)(better-sqlite3@12.10.0)(kysely@0.29.2)(mysql2@3.22.4(@types/node@25.5.0)): + optionalDependencies: + '@opentelemetry/api': 1.9.0 + '@types/better-sqlite3': 7.6.13 + '@types/pg': 8.15.4 + better-sqlite3: 12.10.0 + kysely: 0.29.2 + mysql2: 3.22.4(@types/node@25.5.0) + ebnf@1.9.1: {} ecdsa-sig-formatter@1.0.11: @@ -6333,6 +7231,60 @@ snapshots: es-module-lexer@2.0.0: {} + esbuild@0.18.20: + optionalDependencies: + '@esbuild/android-arm': 0.18.20 + '@esbuild/android-arm64': 0.18.20 + '@esbuild/android-x64': 0.18.20 + '@esbuild/darwin-arm64': 0.18.20 + '@esbuild/darwin-x64': 0.18.20 + '@esbuild/freebsd-arm64': 0.18.20 + '@esbuild/freebsd-x64': 0.18.20 + '@esbuild/linux-arm': 0.18.20 + '@esbuild/linux-arm64': 0.18.20 + '@esbuild/linux-ia32': 0.18.20 + '@esbuild/linux-loong64': 0.18.20 + '@esbuild/linux-mips64el': 0.18.20 + '@esbuild/linux-ppc64': 0.18.20 + '@esbuild/linux-riscv64': 0.18.20 + '@esbuild/linux-s390x': 0.18.20 + '@esbuild/linux-x64': 0.18.20 + '@esbuild/netbsd-x64': 0.18.20 + '@esbuild/openbsd-x64': 0.18.20 + '@esbuild/sunos-x64': 0.18.20 + '@esbuild/win32-arm64': 0.18.20 + '@esbuild/win32-ia32': 0.18.20 + '@esbuild/win32-x64': 0.18.20 + + esbuild@0.25.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.12 + '@esbuild/android-arm': 0.25.12 + '@esbuild/android-arm64': 0.25.12 + '@esbuild/android-x64': 0.25.12 + '@esbuild/darwin-arm64': 0.25.12 + '@esbuild/darwin-x64': 0.25.12 + '@esbuild/freebsd-arm64': 0.25.12 + '@esbuild/freebsd-x64': 0.25.12 + '@esbuild/linux-arm': 0.25.12 + '@esbuild/linux-arm64': 0.25.12 + '@esbuild/linux-ia32': 0.25.12 + '@esbuild/linux-loong64': 0.25.12 + '@esbuild/linux-mips64el': 0.25.12 + '@esbuild/linux-ppc64': 0.25.12 + '@esbuild/linux-riscv64': 0.25.12 + '@esbuild/linux-s390x': 0.25.12 + '@esbuild/linux-x64': 0.25.12 + '@esbuild/netbsd-arm64': 0.25.12 + '@esbuild/netbsd-x64': 0.25.12 + '@esbuild/openbsd-arm64': 0.25.12 + '@esbuild/openbsd-x64': 0.25.12 + '@esbuild/openharmony-arm64': 0.25.12 + '@esbuild/sunos-x64': 0.25.12 + '@esbuild/win32-arm64': 0.25.12 + '@esbuild/win32-ia32': 0.25.12 + '@esbuild/win32-x64': 0.25.12 + esbuild@0.27.0: optionalDependencies: '@esbuild/aix-ppc64': 0.27.0 @@ -6362,6 +7314,35 @@ snapshots: '@esbuild/win32-ia32': 0.27.0 '@esbuild/win32-x64': 0.27.0 + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + escalade@3.1.2: {} esprima@4.0.1: {} @@ -6534,6 +7515,10 @@ snapshots: '@sec-ant/readable-stream': 0.4.1 is-stream: 4.0.1 + get-tsconfig@4.14.0: + dependencies: + resolve-pkg-maps: 1.0.0 + github-from-package@0.0.0: {} glob-parent@5.1.2: @@ -7386,6 +8371,8 @@ snapshots: resolve-from@5.0.0: {} + resolve-pkg-maps@1.0.0: {} + resolve@1.22.8: dependencies: is-core-module: 2.14.0 @@ -7547,6 +8534,13 @@ snapshots: source-map-js@1.2.1: {} + source-map-support@0.5.21: + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + + source-map@0.6.1: {} + sparse-bitfield@3.0.3: dependencies: memory-pager: 1.5.0 @@ -7724,6 +8718,12 @@ snapshots: tslib@2.8.1: {} + tsx@4.23.0: + dependencies: + esbuild: 0.28.1 + optionalDependencies: + fsevents: 2.3.3 + tunnel-agent@0.6.0: dependencies: safe-buffer: 5.2.1 @@ -7766,7 +8766,7 @@ snapshots: vary@1.1.2: {} - vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3): + vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -7775,11 +8775,12 @@ snapshots: tinyglobby: 0.2.16 optionalDependencies: '@types/node': 22.16.2 - esbuild: 0.27.0 + esbuild: 0.28.1 fsevents: 2.3.3 + tsx: 4.23.0 yaml: 2.8.3 - vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3): + vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -7788,14 +8789,15 @@ snapshots: tinyglobby: 0.2.16 optionalDependencies: '@types/node': 25.5.0 - esbuild: 0.27.0 + esbuild: 0.28.1 fsevents: 2.3.3 + tsx: 4.23.0 yaml: 2.8.3 - vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)): + vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.5 - '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) '@vitest/pretty-format': 4.1.5 '@vitest/runner': 4.1.5 '@vitest/snapshot': 4.1.5 @@ -7812,7 +8814,7 @@ snapshots: tinyexec: 1.0.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 @@ -7822,10 +8824,10 @@ snapshots: transitivePeerDependencies: - msw - vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)): + vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.5 - '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) + '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) '@vitest/pretty-format': 4.1.5 '@vitest/runner': 4.1.5 '@vitest/snapshot': 4.1.5 @@ -7842,7 +8844,7 @@ snapshots: tinyexec: 1.0.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 diff --git a/service/Dockerfile b/service/Dockerfile index 71726a51f..8703dea04 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -22,6 +22,7 @@ COPY modules/module-postgres-storage/package.json modules/module-postgres-storag COPY modules/module-mongodb/package.json modules/module-mongodb/tsconfig.json modules/module-mongodb/ COPY modules/module-mongodb-storage/package.json modules/module-mongodb-storage/tsconfig.json modules/module-mongodb-storage/ COPY modules/module-mikroorm-storage/package.json modules/module-mikroorm-storage/tsconfig.json modules/module-mikroorm-storage/ +COPY modules/module-drizzle-storage/package.json modules/module-drizzle-storage/tsconfig.json modules/module-drizzle-storage/ COPY modules/module-mysql/package.json modules/module-mysql/tsconfig.json modules/module-mysql/ COPY modules/module-mssql/package.json modules/module-mssql/tsconfig.json modules/module-mssql/ COPY modules/module-convex/package.json modules/module-convex/tsconfig.json modules/module-convex/ @@ -52,6 +53,7 @@ COPY modules/module-postgres-storage/src modules/module-postgres-storage/src/ COPY modules/module-mongodb/src modules/module-mongodb/src/ COPY modules/module-mongodb-storage/src modules/module-mongodb-storage/src/ COPY modules/module-mikroorm-storage/src modules/module-mikroorm-storage/src/ +COPY modules/module-drizzle-storage/src modules/module-drizzle-storage/src/ COPY modules/module-mysql/src modules/module-mysql/src/ COPY modules/module-mssql/src modules/module-mssql/src/ COPY modules/module-convex/src modules/module-convex/src/ @@ -60,6 +62,8 @@ RUN pnpm build:production && \ rm -rf node_modules **/node_modules && \ pnpm install --frozen-lockfile --prod --ignore-scripts && \ cd modules/module-mikroorm-storage && \ + pnpm rebuild better-sqlite3 && \ + cd ../module-drizzle-storage && \ pnpm rebuild better-sqlite3 # === PROD === From 644a9dd9153b9ffa8c7972e9a03f401f210ac38e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 16 Jul 2026 11:18:50 +0200 Subject: [PATCH 08/11] wip: drizzle performance improvements --- .../src/drivers/sqlite/schema.ts | 7 + .../src/drivers/sqlite/sqlite-dialect.ts | 6 +- .../sqlite/0001_true_matthew_murdock.sql | 14 + .../migrations/sqlite/meta/0001_snapshot.json | 579 ++++ .../src/migrations/sqlite/meta/_journal.json | 7 + .../src/storage/DrizzleBucketBatch.ts | 54 +- .../src/storage/DrizzleStorageDialect.ts | 4 +- .../src/storage/DrizzleSyncRulesStorage.ts | 98 +- .../__snapshots__/storage_sync.test.ts.snap | 2378 ----------------- .../test/src/migrations.test.ts | 4 + .../module-drizzle-storage/test/src/util.ts | 4 +- .../module-postgres-storage/test/src/util.ts | 4 +- 12 files changed, 716 insertions(+), 2443 deletions(-) create mode 100644 modules/module-drizzle-storage/src/migrations/sqlite/0001_true_matthew_murdock.sql create mode 100644 modules/module-drizzle-storage/src/migrations/sqlite/meta/0001_snapshot.json diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts b/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts index 6c6619592..fe3e57e92 100644 --- a/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts +++ b/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts @@ -68,6 +68,11 @@ export const instance = sqliteTable('instance', { id: text('id').primaryKey() }); +export const opIdSequence = sqliteTable('op_id_sequence', { + id: integer('id').primaryKey(), + nextOpId: sqliteBigInt('next_op_id').notNull() +}); + export const sourceTables = sqliteTable( 'source_tables', { @@ -123,6 +128,7 @@ export const sqliteSchema = { bucketParameters, currentData, instance, + opIdSequence, sourceTables, syncRules, writeCheckpoints @@ -132,6 +138,7 @@ export type BucketDataRow = typeof bucketData.$inferSelect; export type BucketParametersRow = typeof bucketParameters.$inferSelect; export type CurrentDataRow = typeof currentData.$inferSelect; export type InstanceRow = typeof instance.$inferSelect; +export type OpIdSequenceRow = typeof opIdSequence.$inferSelect; export type SourceTableRow = typeof sourceTables.$inferSelect; export type SyncRulesRow = typeof syncRules.$inferSelect; export type WriteCheckpointRow = typeof writeCheckpoints.$inferSelect; diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts index 8312b7b88..17478c67e 100644 --- a/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts +++ b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts @@ -21,7 +21,7 @@ export function createSqliteDrizzleStorageDialect(runtime: SqliteDrizzleRuntime) const statement = db.$client .prepare( ` - SELECT id, group_id, bucket_name, op_id, op, source_table, source_key, + SELECT bucket_name, op_id, op, source_table, source_key, table_name, row_id, checksum, data, target_op FROM bucket_data WHERE group_id = ? AND bucket_name = ? AND op_id > ? AND op_id <= ? @@ -52,8 +52,6 @@ export function createSqliteDrizzleStorageDialect(runtime: SqliteDrizzleRuntime) } interface RawBucketDataRow { - id: string; - group_id: bigint; bucket_name: string; op_id: bigint; op: string; @@ -68,8 +66,6 @@ interface RawBucketDataRow { function mapBucketDataRow(row: RawBucketDataRow) { return { - id: row.id, - groupId: Number(row.group_id), bucketName: row.bucket_name, opId: row.op_id, op: row.op, diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/0001_true_matthew_murdock.sql b/modules/module-drizzle-storage/src/migrations/sqlite/0001_true_matthew_murdock.sql new file mode 100644 index 000000000..9cafe1d5b --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/0001_true_matthew_murdock.sql @@ -0,0 +1,14 @@ +CREATE TABLE `op_id_sequence` ( + `id` integer PRIMARY KEY NOT NULL, + `next_op_id` bigint NOT NULL +); +--> statement-breakpoint +INSERT INTO `op_id_sequence` (`id`, `next_op_id`) +SELECT 1, MAX(`max_op_id`) + 1 +FROM ( + SELECT COALESCE(MAX(`op_id`), 0) AS `max_op_id` FROM `bucket_data` + UNION ALL + SELECT COALESCE(MAX(`id`), 0) AS `max_op_id` FROM `bucket_parameters` + UNION ALL + SELECT COALESCE(MAX(`pending_delete`), 0) AS `max_op_id` FROM `current_data` +); diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/meta/0001_snapshot.json b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0001_snapshot.json new file mode 100644 index 000000000..dd6724aad --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0001_snapshot.json @@ -0,0 +1,579 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "c3a62738-6f52-44ca-b288-5afcb936b63e", + "prevId": "d595e266-c1de-43fe-9e46-2deb2b424122", + "tables": { + "bucket_data": { + "name": "bucket_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_name": { + "name": "bucket_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op_id": { + "name": "op_id", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op": { + "name": "op", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checksum": { + "name": "checksum", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_op": { + "name": "target_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bucket_data_bucket_op_index": { + "name": "bucket_data_bucket_op_index", + "columns": ["group_id", "bucket_name", "op_id"], + "isUnique": false + }, + "bucket_data_source_index": { + "name": "bucket_data_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bucket_parameters": { + "name": "bucket_parameters", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookup": { + "name": "lookup", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_parameters": { + "name": "bucket_parameters", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bucket_parameters_lookup_index": { + "name": "bucket_parameters_lookup_index", + "columns": ["group_id", "lookup", "id"], + "isUnique": false + }, + "bucket_parameters_source_index": { + "name": "bucket_parameters_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "current_data": { + "name": "current_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "buckets": { + "name": "buckets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookups": { + "name": "lookups", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pending_delete": { + "name": "pending_delete", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "current_data_source_index": { + "name": "current_data_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + }, + "current_data_pending_delete_index": { + "name": "current_data_pending_delete_index", + "columns": ["group_id", "pending_delete"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance": { + "name": "instance", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "op_id_sequence": { + "name": "op_id_sequence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "next_op_id": { + "name": "next_op_id", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "source_tables": { + "name": "source_tables", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connection_id": { + "name": "connection_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relation_id": { + "name": "relation_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schema_name": { + "name": "schema_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "replica_id_columns": { + "name": "replica_id_columns", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "snapshot_total_estimated_count": { + "name": "snapshot_total_estimated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_replicated_count": { + "name": "snapshot_replicated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_last_key": { + "name": "snapshot_last_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "source_table_lookup": { + "name": "source_table_lookup", + "columns": ["group_id", "table_name"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sync_rules": { + "name": "sync_rules", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "snapshot_lsn": { + "name": "snapshot_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint": { + "name": "last_checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint_lsn": { + "name": "last_checkpoint_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "no_checkpoint_before": { + "name": "no_checkpoint_before", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "slot_name": { + "name": "slot_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_checkpoint_ts": { + "name": "last_checkpoint_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_keepalive_ts": { + "name": "last_keepalive_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error": { + "name": "last_fatal_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error_ts": { + "name": "last_fatal_error_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "keepalive_op": { + "name": "keepalive_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storage_version": { + "name": "storage_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sync_plan": { + "name": "sync_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "write_checkpoints": { + "name": "write_checkpoints", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "sync_rules_id": { + "name": "sync_rules_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checkpoint": { + "name": "checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "heads": { + "name": "heads", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "write_checkpoints_user_checkpoint_index": { + "name": "write_checkpoints_user_checkpoint_index", + "columns": ["user_id", "sync_rules_id", "checkpoint"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json b/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json index 35ed86f69..b7ff902f6 100644 --- a/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json +++ b/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1783955536218, "tag": "0000_real_harpoon", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1784130979161, + "tag": "0001_true_matthew_murdock", + "breakpoints": true } ] } diff --git a/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts b/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts index 81adf952a..6711ed828 100644 --- a/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts +++ b/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts @@ -1,7 +1,7 @@ import { BaseObserver, DO_NOT_LOG, Logger, ReplicationAssertionError } from '@powersync/lib-services-framework'; import { ColumnDescriptor, storage, utils } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; -import { and, count, desc, eq, inArray, isNotNull, isNull, lte, ne } from 'drizzle-orm'; +import { and, count, eq, inArray, isNull, lte, ne } from 'drizzle-orm'; import * as uuid from 'uuid'; import type { SourceTableRow } from '../drivers/sqlite/schema.js'; import type { DrizzleStorageTransaction } from '../drivers/sqlite/sqlite-config.js'; @@ -183,10 +183,12 @@ export class DrizzleBucketBatch async truncate(sourceTables: storage.SourceTable[]): Promise { await this.flush(); - let nextOpId = this.getNextOpId(); - const firstOpId = nextOpId; + let nextOpId = 0n; + let firstOpId = 0n; this.options.dialect.transaction((tx) => { + nextOpId = this.getNextOpId(tx); + firstOpId = nextOpId; const persistedBatch = this.createPersistedBatch(tx); for (const table of sourceTables) { if (!table.syncData && !table.syncParameters) { @@ -235,6 +237,7 @@ export class DrizzleBucketBatch ) .run(); } + this.setNextOpId(tx, nextOpId); }); if (nextOpId == firstOpId) { @@ -275,12 +278,15 @@ export class DrizzleBucketBatch if (this.pendingOperations.length > 0) { await this.options.hooks?.beforeBatchFlush?.(this); const operations = this.pendingOperations.splice(0); - let nextOpId = this.getNextOpId(); - const firstOpId = nextOpId; + let nextOpId = 0n; + let firstOpId: bigint | null = null; for (const batch of chunked(operations, MAX_OPERATION_BATCH_COUNT)) { this.options.dialect.transaction((tx) => { + nextOpId = this.getNextOpId(tx); + firstOpId ??= nextOpId; const persistedBatch = this.createPersistedBatch(tx); nextOpId = persistedBatch.persistOperations(batch, nextOpId); + this.setNextOpId(tx, nextOpId); }); } const lastOpId = nextOpId - 1n; @@ -289,7 +295,7 @@ export class DrizzleBucketBatch result = { flushed_op: lastOpId }; this.logFlushedOperations({ operationCount: operations.length, - firstOpId, + firstOpId: firstOpId!, nextOpId }); await this.options.hooks?.afterBatchFlush?.(this); @@ -571,29 +577,23 @@ export class DrizzleBucketBatch }); } - private getNextOpId(): bigint { - const { db, tables } = this.options.dialect; - const bucketData = db - .select({ value: tables.bucketData.opId }) - .from(tables.bucketData) - .orderBy(desc(tables.bucketData.opId)) - .limit(1) - .get(); - const bucketParameters = db - .select({ value: tables.bucketParameters.id }) - .from(tables.bucketParameters) - .orderBy(desc(tables.bucketParameters.id)) - .limit(1) - .get(); - const pendingDelete = db - .select({ value: tables.currentData.pendingDelete }) - .from(tables.currentData) - .where(isNotNull(tables.currentData.pendingDelete)) - .orderBy(desc(tables.currentData.pendingDelete)) - .limit(1) + private getNextOpId(tx: DrizzleStorageTransaction): bigint { + const row = tx + .select() + .from(this.options.dialect.tables.opIdSequence) + .where(eq(this.options.dialect.tables.opIdSequence.id, 1)) .get(); + if (row == null) { + throw new Error('Missing op ID sequence state'); + } + return row.nextOpId; + } - return maxBigint(bucketData?.value, bucketParameters?.value, pendingDelete?.value, 0n) + 1n; + private setNextOpId(tx: DrizzleStorageTransaction, nextOpId: bigint): void { + tx.update(this.options.dialect.tables.opIdSequence) + .set({ nextOpId }) + .where(eq(this.options.dialect.tables.opIdSequence.id, 1)) + .run(); } private logFlushedOperations(options: { diff --git a/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts b/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts index 1a1346391..7bab0517f 100644 --- a/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts +++ b/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts @@ -7,10 +7,12 @@ export interface DrizzleStorageDialect { readonly db: DrizzleStorageDatabase; readonly tables: typeof sqliteSchema; transaction(callback: (tx: DrizzleStorageTransaction) => T): T; - streamBucketDataRows(options: DrizzleBucketDataStreamOptions): AsyncIterable; + streamBucketDataRows(options: DrizzleBucketDataStreamOptions): AsyncIterable; createCheckpointWatcher(): DrizzleCheckpointWatcher; } +export type BucketDataReadRow = Omit; + export interface DrizzleBucketDataStreamOptions { readonly db?: DrizzleStorageDatabase; readonly groupId: number; diff --git a/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts b/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts index d79e3070e..54f6e8f06 100644 --- a/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts +++ b/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts @@ -17,13 +17,13 @@ import { } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; import * as sync_rules from '@powersync/service-sync-rules'; -import { and, asc, desc, eq, isNull, lte } from 'drizzle-orm'; +import { and, count, desc, eq, gt, isNull, lte, sum } from 'drizzle-orm'; import * as uuid from 'uuid'; import type { BucketDataRow } from '../drivers/sqlite/schema.js'; import { DrizzleBucketBatch } from './DrizzleBucketBatch.js'; import { DrizzleBucketStorageFactory } from './DrizzleBucketStorageFactory.js'; import { DrizzleCompactor } from './DrizzleCompactor.js'; -import { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; +import { BucketDataReadRow, DrizzleStorageDialect } from './DrizzleStorageDialect.js'; export interface DrizzleSyncRulesStorageOptions { factory: DrizzleBucketStorageFactory; @@ -48,6 +48,7 @@ export class DrizzleSyncRulesStorage | undefined; private writeCheckpointModeValue = storage.WriteCheckpointMode.MANAGED; + private checksumCacheValue: storage.ChecksumCache | undefined; constructor(private readonly options: DrizzleSyncRulesStorageOptions) { super(); @@ -66,6 +67,13 @@ export class DrizzleSyncRulesStorage this.writeCheckpointModeValue = mode; } + private get checksumCache(): storage.ChecksumCache { + this.checksumCacheValue ??= new storage.ChecksumCache({ + fetchChecksums: (batch) => this.getChecksumsInternal(batch) + }); + return this.checksumCacheValue; + } + async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) { throw new errors.ValidationError( @@ -208,6 +216,7 @@ export class DrizzleSyncRulesStorage tx.delete(tables.sourceTables).where(eq(tables.sourceTables.groupId, this.replicationStreamId)).run(); }); + this.clearChecksumCache(); this.factory.checkpointWatcher.notify(); } @@ -239,6 +248,10 @@ export class DrizzleSyncRulesStorage if (options?.compactParameterData) { await compactor.compactParameterData(options); } + + // Compaction can replace operations at an already-cached checkpoint with a + // CLEAR operation, invalidating any incremental checksum based on it. + this.clearChecksumCache(); } async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise { @@ -399,34 +412,59 @@ export class DrizzleSyncRulesStorage } async getChecksums(checkpoint: bigint, buckets: BucketChecksumRequest[]): Promise { - const result: utils.ChecksumMap = new Map(); + return this.checksumCache.getChecksumMap(checkpoint, buckets); + } + + clearChecksumCache(): void { + this.checksumCacheValue?.clear(); + } + + private async getChecksumsInternal(batch: storage.FetchPartialBucketChecksum[]): Promise { + const result: storage.PartialChecksumMap = new Map(); + if (batch.length == 0) { + return result; + } + const { db, tables } = this.options.dialect; - for (const bucket of buckets) { - const rows = db - .select() + + for (const request of batch) { + const rangeFilter = and( + eq(tables.bucketData.groupId, this.replicationStreamId), + eq(tables.bucketData.bucketName, request.bucket), + gt(tables.bucketData.opId, request.start ?? 0n), + lte(tables.bucketData.opId, request.end) + ); + const aggregate = db + .select({ + checksum: sum(tables.bucketData.checksum), + count: count() + }) .from(tables.bucketData) - .where( - and( - eq(tables.bucketData.groupId, this.replicationStreamId), - eq(tables.bucketData.bucketName, bucket.bucket), - lte(tables.bucketData.opId, checkpoint) - ) - ) - .orderBy(asc(tables.bucketData.opId)) - .all(); - const entries = rows.map(bucketDataRowToOpEntry); - const checksum = entries.reduce((total, entry) => utils.addChecksums(total, Number(entry.checksum)), 0); - result.set(bucket.bucket, { - bucket: bucket.bucket, - checksum, - count: entries.length - }); + .where(rangeFilter) + .get(); + const checksum = Number(BigInt(aggregate?.checksum ?? 0) & 0xffffffffn) & 0xffffffff; + const rowCount = aggregate?.count ?? 0; + + // A CLEAR only changes how a partial result is combined with an existing + // cached checksum. Full requests are already combined with an empty base. + const hasClearOperation = + request.start != null && + db + .select({ opId: tables.bucketData.opId }) + .from(tables.bucketData) + .where(and(rangeFilter, eq(tables.bucketData.op, 'CLEAR'))) + .limit(1) + .get() != null; + + result.set( + request.bucket, + hasClearOperation + ? { bucket: request.bucket, checksum, count: rowCount } + : { bucket: request.bucket, partialChecksum: checksum, partialCount: rowCount } + ); } - return result; - } - clearChecksumCache(): void { - // No checksum cache exists in the initial Drizzle storage slice. + return result; } private async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters): Promise { @@ -510,8 +548,12 @@ export class DrizzleSyncRulesStorage } const { db, tables } = this.options.dialect; + // Luckily, we usually only have one record per user here const rows = db - .select() + .select({ + checkpoint: tables.writeCheckpoints.checkpoint, + heads: tables.writeCheckpoints.heads + }) .from(tables.writeCheckpoints) .where(and(eq(tables.writeCheckpoints.userId, filters.user_id), isNull(tables.writeCheckpoints.syncRulesId))) .orderBy(desc(tables.writeCheckpoints.checkpoint)) @@ -542,7 +584,7 @@ function parseBucketParameters(value: unknown): sync_rules.SqliteJsonRow[] { return Array.isArray(value) ? (value as sync_rules.SqliteJsonRow[]) : []; } -function bucketDataRowToOpEntry(row: BucketDataRow): utils.OplogEntry { +function bucketDataRowToOpEntry(row: BucketDataRow | BucketDataReadRow): utils.OplogEntry { if (row.op == 'PUT' || row.op == 'REMOVE') { return { op_id: utils.internalToExternalOpId(row.opId), diff --git a/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap index db64ec935..511e185fd 100644 --- a/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -1188,2381 +1188,3 @@ exports[`sync - Drizzle SQLite > storage v1 > sync updates to parameter query on }, ] `; - -exports[`sync - Drizzle SQLite > storage v2 > can override priority when subscribing to stream 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#todos|0["a"]", - "checksum": -1712802421, - "count": 1, - "priority": 0, - "subscriptions": [ - { - "sub": 0, - }, - { - "sub": 1, - }, - ], - }, - { - "bucket": "1#todos|0["b"]", - "checksum": -1291414318, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "sub": 1, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": false, - "name": "todos", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#todos|0["a"]", - "data": [ - { - "checksum": 2582164875, - "data": "{"id":"a","description":"Test 1"}", - "object_id": "a", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "partial_checkpoint_complete": { - "last_op_id": "2", - "priority": 0, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#todos|0["b"]", - "data": [ - { - "checksum": 3003552978, - "data": "{"id":"b","description":"Test 2"}", - "object_id": "b", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > compacting data - invalidate checkpoint 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -93886621, - "count": 2, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > compacting data - invalidate checkpoint 2`] = ` -[ - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": -93886621, - "op": "CLEAR", - "op_id": "2", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_diff": { - "last_op_id": "4", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": 499012468, - "count": 3, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "2", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 1859363232, - "data": "{"id":"t1","description":"Test 1b"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "3", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - { - "checksum": 3028503153, - "data": "{"id":"t2","description":"Test 2b"}", - "object_id": "t2", - "object_type": "test", - "op": "PUT", - "op_id": "4", - "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", - }, - ], - "has_more": false, - "next_after": "4", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "4", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > encodes sync rules id in buckets for streams 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#test|0[]", - "checksum": 920318466, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "1", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "test", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#test|0[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > encodes sync rules id in buckets for streams 2`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "2#test2|0[]", - "checksum": 920318466, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "test2", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "2#test2|0[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > expired token 1`] = ` -[ - { - "token_expires_in": 0, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > expiring token 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": 0, - "count": 0, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "0", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "0", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > expiring token 2`] = ` -[ - { - "token_expires_in": 0, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sends checkpoint complete line for empty checkpoint 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -1221282404, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "1", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 3073684892, - "data": "{"id":"t1","description":"sync"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - null, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, - { - "checkpoint_diff": { - "last_op_id": "1", - "removed_buckets": [], - "updated_buckets": [], - "write_checkpoint": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync buckets in order 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#b0[]", - "checksum": 920318466, - "count": 1, - "priority": 2, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - { - "bucket": "1#b1[]", - "checksum": -1382098757, - "count": 1, - "priority": 1, - "subscriptions": [ - { - "default": 1, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "b0", - }, - { - "errors": [], - "is_default": true, - "name": "b1", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b1[]", - "data": [ - { - "checksum": 2912868539, - "data": "{"id":"earlier","description":"Test 2"}", - "object_id": "earlier", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "partial_checkpoint_complete": { - "last_op_id": "2", - "priority": 1, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b0[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync global data 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -93886621, - "count": 2, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - { - "checksum": 3280762209, - "data": "{"id":"t2","description":"Test 2"}", - "object_id": "t2", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#b0a[]", - "checksum": -659831575, - "count": 2000, - "priority": 2, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - { - "bucket": "1#b0b[]", - "checksum": -659831575, - "count": 2000, - "priority": 2, - "subscriptions": [ - { - "default": 1, - }, - ], - }, - { - "bucket": "1#b1[]", - "checksum": -1096116670, - "count": 1, - "priority": 1, - "subscriptions": [ - { - "default": 2, - }, - ], - }, - ], - "last_op_id": "4001", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "b0a", - }, - { - "errors": [], - "is_default": true, - "name": "b0b", - }, - { - "errors": [], - "is_default": true, - "name": "b1", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b1[]", - "data": undefined, - "has_more": false, - "next_after": "1", - }, - }, - { - "partial_checkpoint_complete": { - "last_op_id": "4001", - "priority": 1, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b0a[]", - "data": undefined, - "has_more": true, - "next_after": "2000", - }, - }, - { - "data": { - "after": "2000", - "bucket": "1#b0a[]", - "data": undefined, - "has_more": true, - "next_after": "4000", - }, - }, - { - "checkpoint_diff": { - "last_op_id": "4004", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#b0a[]", - "checksum": 883076828, - "count": 2001, - "priority": 2, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - { - "bucket": "1#b0b[]", - "checksum": 883076828, - "count": 2001, - "priority": 2, - "subscriptions": [ - { - "default": 1, - }, - ], - }, - { - "bucket": "1#b1[]", - "checksum": 1841937527, - "count": 2, - "priority": 1, - "subscriptions": [ - { - "default": 2, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "1", - "bucket": "1#b1[]", - "data": undefined, - "has_more": false, - "next_after": "4002", - }, - }, - { - "partial_checkpoint_complete": { - "last_op_id": "4004", - "priority": 1, - }, - }, - { - "data": { - "after": "4000", - "bucket": "1#b0a[]", - "data": undefined, - "has_more": false, - "next_after": "4003", - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b0b[]", - "data": undefined, - "has_more": true, - "next_after": "1999", - }, - }, - { - "data": { - "after": "1999", - "bucket": "1#b0b[]", - "data": undefined, - "has_more": true, - "next_after": "3999", - }, - }, - { - "data": { - "after": "3999", - "bucket": "1#b0b[]", - "data": undefined, - "has_more": false, - "next_after": "4004", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "4004", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync legacy non-raw data 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -852817836, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "1", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 3442149460n, - "data": { - "description": "Test -"string"", - "id": "t1", - "large_num": 12345678901234567890n, - }, - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync updates to data query only 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#by_user["user1"]", - "checksum": 0, - "count": 0, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "1", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "by_user", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync updates to data query only 2`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "2", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#by_user["user1"]", - "checksum": 1418351250, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#by_user["user1"]", - "data": [ - { - "checksum": 1418351250, - "data": "{"id":"list1","user_id":"user1","name":"User 1"}", - "object_id": "list1", - "object_type": "lists", - "op": "PUT", - "op_id": "2", - "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync updates to global data 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": 0, - "count": 0, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "0", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "0", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync updates to global data 2`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "1", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": 920318466, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync updates to global data 3`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "2", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -93886621, - "count": 2, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "1", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 3280762209, - "data": "{"id":"t2","description":"Test 2"}", - "object_id": "t2", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync updates to parameter query + data 1`] = ` -[ - { - "checkpoint": { - "buckets": [], - "last_op_id": "0", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "by_user", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "0", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync updates to parameter query + data 2`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "2", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#by_user["user1"]", - "checksum": 1418351250, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#by_user["user1"]", - "data": [ - { - "checksum": 1418351250, - "data": "{"id":"list1","user_id":"user1","name":"User 1"}", - "object_id": "list1", - "object_type": "lists", - "op": "PUT", - "op_id": "1", - "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync updates to parameter query only 1`] = ` -[ - { - "checkpoint": { - "buckets": [], - "last_op_id": "0", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "by_user", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "0", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v2 > sync updates to parameter query only 2`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "1", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#by_user["user1"]", - "checksum": 0, - "count": 0, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > can override priority when subscribing to stream 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#todos|0["a"]", - "checksum": -1712802421, - "count": 1, - "priority": 0, - "subscriptions": [ - { - "sub": 0, - }, - { - "sub": 1, - }, - ], - }, - { - "bucket": "1#todos|0["b"]", - "checksum": -1291414318, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "sub": 1, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": false, - "name": "todos", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#todos|0["a"]", - "data": [ - { - "checksum": 2582164875, - "data": "{"id":"a","description":"Test 1"}", - "object_id": "a", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "partial_checkpoint_complete": { - "last_op_id": "2", - "priority": 0, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#todos|0["b"]", - "data": [ - { - "checksum": 3003552978, - "data": "{"id":"b","description":"Test 2"}", - "object_id": "b", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > compacting data - invalidate checkpoint 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -93886621, - "count": 2, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > compacting data - invalidate checkpoint 2`] = ` -[ - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": -93886621, - "op": "CLEAR", - "op_id": "2", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_diff": { - "last_op_id": "4", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": 499012468, - "count": 3, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "2", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 1859363232, - "data": "{"id":"t1","description":"Test 1b"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "3", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - { - "checksum": 3028503153, - "data": "{"id":"t2","description":"Test 2b"}", - "object_id": "t2", - "object_type": "test", - "op": "PUT", - "op_id": "4", - "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", - }, - ], - "has_more": false, - "next_after": "4", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "4", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > encodes sync rules id in buckets for streams 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#test|0[]", - "checksum": 920318466, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "1", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "test", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#test|0[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > encodes sync rules id in buckets for streams 2`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "2#test2|0[]", - "checksum": 920318466, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "test2", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "2#test2|0[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > expired token 1`] = ` -[ - { - "token_expires_in": 0, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > expiring token 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": 0, - "count": 0, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "0", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "0", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > expiring token 2`] = ` -[ - { - "token_expires_in": 0, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sends checkpoint complete line for empty checkpoint 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -1221282404, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "1", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 3073684892, - "data": "{"id":"t1","description":"sync"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - null, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, - { - "checkpoint_diff": { - "last_op_id": "1", - "removed_buckets": [], - "updated_buckets": [], - "write_checkpoint": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync buckets in order 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#b0[]", - "checksum": 920318466, - "count": 1, - "priority": 2, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - { - "bucket": "1#b1[]", - "checksum": -1382098757, - "count": 1, - "priority": 1, - "subscriptions": [ - { - "default": 1, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "b0", - }, - { - "errors": [], - "is_default": true, - "name": "b1", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b1[]", - "data": [ - { - "checksum": 2912868539, - "data": "{"id":"earlier","description":"Test 2"}", - "object_id": "earlier", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "partial_checkpoint_complete": { - "last_op_id": "2", - "priority": 1, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b0[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync global data 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -93886621, - "count": 2, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "2", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - { - "checksum": 3280762209, - "data": "{"id":"t2","description":"Test 2"}", - "object_id": "t2", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#b0a[]", - "checksum": -659831575, - "count": 2000, - "priority": 2, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - { - "bucket": "1#b0b[]", - "checksum": -659831575, - "count": 2000, - "priority": 2, - "subscriptions": [ - { - "default": 1, - }, - ], - }, - { - "bucket": "1#b1[]", - "checksum": -1096116670, - "count": 1, - "priority": 1, - "subscriptions": [ - { - "default": 2, - }, - ], - }, - ], - "last_op_id": "4001", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "b0a", - }, - { - "errors": [], - "is_default": true, - "name": "b0b", - }, - { - "errors": [], - "is_default": true, - "name": "b1", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b1[]", - "data": undefined, - "has_more": false, - "next_after": "1", - }, - }, - { - "partial_checkpoint_complete": { - "last_op_id": "4001", - "priority": 1, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b0a[]", - "data": undefined, - "has_more": true, - "next_after": "2000", - }, - }, - { - "data": { - "after": "2000", - "bucket": "1#b0a[]", - "data": undefined, - "has_more": true, - "next_after": "4000", - }, - }, - { - "checkpoint_diff": { - "last_op_id": "4004", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#b0a[]", - "checksum": 883076828, - "count": 2001, - "priority": 2, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - { - "bucket": "1#b0b[]", - "checksum": 883076828, - "count": 2001, - "priority": 2, - "subscriptions": [ - { - "default": 1, - }, - ], - }, - { - "bucket": "1#b1[]", - "checksum": 1841937527, - "count": 2, - "priority": 1, - "subscriptions": [ - { - "default": 2, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "1", - "bucket": "1#b1[]", - "data": undefined, - "has_more": false, - "next_after": "4002", - }, - }, - { - "partial_checkpoint_complete": { - "last_op_id": "4004", - "priority": 1, - }, - }, - { - "data": { - "after": "4000", - "bucket": "1#b0a[]", - "data": undefined, - "has_more": false, - "next_after": "4003", - }, - }, - { - "data": { - "after": "0", - "bucket": "1#b0b[]", - "data": undefined, - "has_more": true, - "next_after": "1999", - }, - }, - { - "data": { - "after": "1999", - "bucket": "1#b0b[]", - "data": undefined, - "has_more": true, - "next_after": "3999", - }, - }, - { - "data": { - "after": "3999", - "bucket": "1#b0b[]", - "data": undefined, - "has_more": false, - "next_after": "4004", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "4004", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync legacy non-raw data 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -852817836, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "1", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 3442149460n, - "data": { - "description": "Test -"string"", - "id": "t1", - "large_num": 12345678901234567890n, - }, - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync updates to data query only 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#by_user["user1"]", - "checksum": 0, - "count": 0, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "1", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "by_user", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync updates to data query only 2`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "2", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#by_user["user1"]", - "checksum": 1418351250, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#by_user["user1"]", - "data": [ - { - "checksum": 1418351250, - "data": "{"id":"list1","user_id":"user1","name":"User 1"}", - "object_id": "list1", - "object_type": "lists", - "op": "PUT", - "op_id": "2", - "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync updates to global data 1`] = ` -[ - { - "checkpoint": { - "buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": 0, - "count": 0, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "last_op_id": "0", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "mybucket", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "0", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync updates to global data 2`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "1", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": 920318466, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 920318466, - "data": "{"id":"t1","description":"Test 1"}", - "object_id": "t1", - "object_type": "test", - "op": "PUT", - "op_id": "1", - "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync updates to global data 3`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "2", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#mybucket[]", - "checksum": -93886621, - "count": 2, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "1", - "bucket": "1#mybucket[]", - "data": [ - { - "checksum": 3280762209, - "data": "{"id":"t2","description":"Test 2"}", - "object_id": "t2", - "object_type": "test", - "op": "PUT", - "op_id": "2", - "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", - }, - ], - "has_more": false, - "next_after": "2", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync updates to parameter query + data 1`] = ` -[ - { - "checkpoint": { - "buckets": [], - "last_op_id": "0", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "by_user", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "0", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync updates to parameter query + data 2`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "2", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#by_user["user1"]", - "checksum": 1418351250, - "count": 1, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "data": { - "after": "0", - "bucket": "1#by_user["user1"]", - "data": [ - { - "checksum": 1418351250, - "data": "{"id":"list1","user_id":"user1","name":"User 1"}", - "object_id": "list1", - "object_type": "lists", - "op": "PUT", - "op_id": "1", - "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", - }, - ], - "has_more": false, - "next_after": "1", - }, - }, - { - "checkpoint_complete": { - "last_op_id": "2", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync updates to parameter query only 1`] = ` -[ - { - "checkpoint": { - "buckets": [], - "last_op_id": "0", - "streams": [ - { - "errors": [], - "is_default": true, - "name": "by_user", - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "0", - }, - }, -] -`; - -exports[`sync - Drizzle SQLite > storage v3 > sync updates to parameter query only 2`] = ` -[ - { - "checkpoint_diff": { - "last_op_id": "1", - "removed_buckets": [], - "updated_buckets": [ - { - "bucket": "1#by_user["user1"]", - "checksum": 0, - "count": 0, - "priority": 3, - "subscriptions": [ - { - "default": 0, - }, - ], - }, - ], - "write_checkpoint": undefined, - }, - }, - { - "checkpoint_complete": { - "last_op_id": "1", - }, - }, -] -`; diff --git a/modules/module-drizzle-storage/test/src/migrations.test.ts b/modules/module-drizzle-storage/test/src/migrations.test.ts index c7681e322..ef4613b70 100644 --- a/modules/module-drizzle-storage/test/src/migrations.test.ts +++ b/modules/module-drizzle-storage/test/src/migrations.test.ts @@ -51,6 +51,7 @@ describe('Drizzle migrations', () => { 'bucket_data', 'bucket_parameters', 'current_data', + 'op_id_sequence', 'source_tables', 'sync_rules', 'write_checkpoints', @@ -58,6 +59,9 @@ describe('Drizzle migrations', () => { 'bucket_parameters_lookup_index' ]) ); + expect(runtime.client.prepare(`SELECT next_op_id FROM op_id_sequence WHERE id = 1`).get()).toEqual({ + next_op_id: 1n + }); } finally { runtime.close(); } diff --git a/modules/module-drizzle-storage/test/src/util.ts b/modules/module-drizzle-storage/test/src/util.ts index bd874b4f9..5fd6010e1 100644 --- a/modules/module-drizzle-storage/test/src/util.ts +++ b/modules/module-drizzle-storage/test/src/util.ts @@ -1,4 +1,4 @@ -import { storage, SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core'; +import { storage } from '@powersync/service-core'; import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; import { createSqliteDrizzleStorageFactory, @@ -24,4 +24,4 @@ export const DRIZZLE_SQLITE_STORAGE_FACTORY: storage.TestStorageConfig = { } }; -export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; +export const TEST_STORAGE_VERSIONS = [storage.LEGACY_STORAGE_VERSION]; diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts index 105214d7f..88dca61ca 100644 --- a/modules/module-postgres-storage/test/src/util.ts +++ b/modules/module-postgres-storage/test/src/util.ts @@ -1,4 +1,4 @@ -import { SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core'; +import { LEGACY_STORAGE_VERSION } from '@powersync/service-core'; import path from 'path'; import { fileURLToPath } from 'url'; import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js'; @@ -36,4 +36,4 @@ export const POSTGRES_STORAGE_SETUP = postgresTestSetup({ export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP; export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory; -export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; +export const TEST_STORAGE_VERSIONS = [LEGACY_STORAGE_VERSION]; From 7588b6ce1e0f1b9a678d2f80c046d8780834c159 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 16 Jul 2026 19:50:10 +0200 Subject: [PATCH 09/11] update after main merge --- .../src/storage/DrizzleBucketBatch.ts | 2 +- .../DrizzlePersistedReplicationStream.ts | 3 +- .../src/storage/DrizzleSyncRulesStorage.ts | 95 ++++++++++-------- .../test/src/sync-rules-storage.test.ts | 21 ++-- .../src/storage/MikroOrmBucketBatch.ts | 2 +- .../MikroOrmPersistedReplicationStream.ts | 3 +- .../src/storage/MikroOrmSyncRulesStorage.ts | 97 ++++++++++--------- .../test/src/sync-rules-storage.test.ts | 21 ++-- 8 files changed, 128 insertions(+), 116 deletions(-) diff --git a/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts b/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts index 6711ed828..5e1c8a2b7 100644 --- a/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts +++ b/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts @@ -67,7 +67,7 @@ export class DrizzleBucketBatch } async resolveTables(options: storage.ResolveTablesOptions): Promise { - const syncRules = options.syncRules ?? this.options.syncRules; + const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.options.syncRules; const { connection_id, source } = options; const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source; const normalizedReplicaIdColumns = normalizeReplicaIdColumns(replicaIdColumns); diff --git a/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts b/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts index ab396bbeb..21070459a 100644 --- a/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts +++ b/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts @@ -15,7 +15,8 @@ export class DrizzlePersistedSyncConfigContent extends storage.PersistedSyncConf compiled_plan: row.syncPlan, replicationStreamName: row.slotName, storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION, - syncConfigId: String(row.id) + syncConfigId: String(row.id), + syncConfigState: row.state as storage.SyncRuleState }); } diff --git a/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts b/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts index 54f6e8f06..8ff903784 100644 --- a/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts +++ b/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts @@ -43,9 +43,7 @@ export class DrizzleSyncRulesStorage readonly factory: DrizzleBucketStorageFactory; readonly logger: Logger; - private parsedSyncRulesCache: - | { parsed: sync_rules.HydratedSyncConfig; options: storage.ParseSyncConfigOptions } - | undefined; + private readonly parsedSyncConfigSets = new Map(); private writeCheckpointModeValue = storage.WriteCheckpointMode.MANAGED; private checksumCacheValue: storage.ChecksumCache | undefined; @@ -74,38 +72,48 @@ export class DrizzleSyncRulesStorage return this.checksumCacheValue; } - async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { + async createManagedWriteCheckpoints( + checkpoints: storage.ManagedWriteCheckpointOptions[] + ): Promise> { if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) { throw new errors.ValidationError( `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` ); } + const uniqueCheckpoints = [...new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint])).values()]; + if (uniqueCheckpoints.length == 0) { + return new Map(); + } const table = this.options.dialect.tables.writeCheckpoints; - const createdCheckpoint = this.options.dialect.transaction((tx) => { - const latest = tx - .select() - .from(table) - .where(and(eq(table.userId, checkpoint.user_id), isNull(table.syncRulesId))) - .orderBy(desc(table.checkpoint)) - .limit(1) - .get(); - const value = (latest?.checkpoint ?? 0n) + 1n; - tx.insert(table) - .values({ - id: uuid.v4(), - syncRulesId: null, - userId: checkpoint.user_id, - checkpoint: value, - heads: checkpoint.heads, - createdAt: new Date() - }) - .run(); - return value; + const createdCheckpoints = this.options.dialect.transaction((tx) => { + const result = new Map(); + for (const checkpoint of uniqueCheckpoints) { + const latest = tx + .select() + .from(table) + .where(and(eq(table.userId, checkpoint.user_id), isNull(table.syncRulesId))) + .orderBy(desc(table.checkpoint)) + .limit(1) + .get(); + const value = (latest?.checkpoint ?? 0n) + 1n; + tx.insert(table) + .values({ + id: uuid.v4(), + syncRulesId: null, + userId: checkpoint.user_id, + checkpoint: value, + heads: checkpoint.heads, + createdAt: new Date() + }) + .run(); + result.set(checkpoint.user_id, value); + } + return result; }); this.factory.checkpointWatcher.notify(); - return createdCheckpoint; + return createdCheckpoints; } async lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { @@ -157,16 +165,17 @@ export class DrizzleSyncRulesStorage return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; } - getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig { - const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; - if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { - this.parsedSyncRulesCache = { - parsed: this.options.replicationStream.parsed(options).hydratedSyncConfig(), - options - }; + getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet { + let parsed = this.parsedSyncConfigSets.get(options.defaultSchema); + if (parsed == null) { + parsed = this.options.replicationStream.parsed(options); + this.parsedSyncConfigSets.set(options.defaultSchema, parsed); } + return parsed; + } - return this.parsedSyncRulesCache!.parsed; + getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig { + return this.getParsedSyncConfigSet(options).hydratedSyncConfig; } async terminate(options?: storage.TerminateOptions): Promise { @@ -185,16 +194,16 @@ export class DrizzleSyncRulesStorage this.factory.checkpointWatcher.notify(); } - async getStatus(): Promise { + async getStatus(): Promise { const { db, tables } = this.options.dialect; const row = db.select().from(tables.syncRules).where(eq(tables.syncRules.id, this.replicationStreamId)).get(); + if (row == null) { + throw new Error('Cannot find replication stream status'); + } return { - checkpoint_lsn: row?.lastCheckpointLsn ?? null, - active: row?.state == storage.SyncRuleState.ACTIVE, - snapshot_done: row?.snapshotDone ?? false, - snapshot_lsn: row?.snapshotLsn ?? null, - keepalive_op: row?.keepaliveOp ?? null + snapshotDone: row.snapshotDone && row.lastCheckpointLsn != null, + resumeLsn: utils.maxLsn(row.snapshotLsn, row.lastCheckpointLsn) }; } @@ -330,7 +339,7 @@ export class DrizzleSyncRulesStorage } async *getBucketDataBatch( - checkpoint: bigint, + checkpoint: ReplicationCheckpoint, dataBuckets: BucketDataRequest[], options?: BucketDataBatchOptions ): AsyncIterable { @@ -343,7 +352,7 @@ export class DrizzleSyncRulesStorage const startOpByBucket = new Map(dataBuckets.map((request) => [request.bucket, request.start])); const rows = this.options.dialect.streamBucketDataRows({ groupId: this.replicationStreamId, - checkpoint, + checkpoint: checkpoint.checkpoint, dataBuckets, limit: batchRowLimit }); @@ -411,8 +420,8 @@ export class DrizzleSyncRulesStorage } } - async getChecksums(checkpoint: bigint, buckets: BucketChecksumRequest[]): Promise { - return this.checksumCache.getChecksumMap(checkpoint, buckets); + async getChecksums(checkpoint: ReplicationCheckpoint, buckets: BucketChecksumRequest[]): Promise { + return this.checksumCache.getChecksumMap(checkpoint.checkpoint, buckets); } clearChecksumCache(): void { diff --git a/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts b/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts index 985e225df..55e6568ca 100644 --- a/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts +++ b/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts @@ -21,14 +21,12 @@ bucket_definitions: const stream = await factory.updateSyncRules(syncRules); const bucketStorage = factory.getInstance(stream); - const first = await bucketStorage.createManagedWriteCheckpoint({ - user_id: 'user1', - heads: { '1': '5/0' } - }); - const second = await bucketStorage.createManagedWriteCheckpoint({ - user_id: 'user1', - heads: { '1': '6/0' } - }); + const first = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '5/0' } }]) + ).get('user1')!; + const second = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '6/0' } }]) + ).get('user1')!; await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '4/0' } })).resolves.toBeNull(); await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '5/0' } })).resolves.toBe(first); @@ -46,10 +44,9 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({ - user_id: 'user1', - heads: { '1': '5/0' } - }); + const writeCheckpoint = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '5/0' } }]) + ).get('user1')!; factory.dialect.db .update(factory.dialect.tables.syncRules) diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts index eaf35cfd5..3ea1b3eaf 100644 --- a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts @@ -67,7 +67,7 @@ export class MikroOrmBucketBatch } async resolveTables(options: storage.ResolveTablesOptions): Promise { - const syncRules = options.syncRules ?? this.options.syncRules; + const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.options.syncRules; const { connection_id, source } = options; const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source; const normalizedReplicaIdColumns = normalizeReplicaIdColumns(replicaIdColumns); diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts index a0ecc6eb1..5db87c451 100644 --- a/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts @@ -16,7 +16,8 @@ export class MikroOrmPersistedSyncConfigContent extends storage.PersistedSyncCon compiled_plan: row.syncPlan, replicationStreamName: row.slotName, storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION, - syncConfigId: String(row.id) + syncConfigId: String(row.id), + syncConfigState: row.state as storage.SyncRuleState }); } diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts index 8aa915930..0e02acb2c 100644 --- a/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts @@ -44,9 +44,7 @@ export class MikroOrmSyncRulesStorage readonly factory: MikroOrmBucketStorageFactory; readonly logger: Logger; - private parsedSyncRulesCache: - | { parsed: sync_rules.HydratedSyncConfig; options: storage.ParseSyncConfigOptions } - | undefined; + private readonly parsedSyncConfigSets = new Map(); private writeCheckpointModeValue = storage.WriteCheckpointMode.MANAGED; @@ -67,43 +65,51 @@ export class MikroOrmSyncRulesStorage this.writeCheckpointModeValue = mode; } - async createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise { + async createManagedWriteCheckpoints( + checkpoints: storage.ManagedWriteCheckpointOptions[] + ): Promise> { if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) { throw new errors.ValidationError( `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` ); } - let createdCheckpoint: bigint | null = null; + const uniqueCheckpoints = [...new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint])).values()]; + if (uniqueCheckpoints.length == 0) { + return new Map(); + } + const createdCheckpoints = new Map(); const em = this.options.orm.em.fork(); await em.transactional(async (transactionalEntityManager) => { - const [latest] = await transactionalEntityManager.find( - this.options.dialect.writeCheckpointEntity, - { + for (const checkpoint of uniqueCheckpoints) { + const [latest] = await transactionalEntityManager.find( + this.options.dialect.writeCheckpointEntity, + { + userId: checkpoint.user_id, + syncRulesId: null + }, + { + orderBy: { checkpoint: 'DESC' }, + limit: 1 + } + ); + const value = (latest?.checkpoint ?? 0n) + 1n; + const row = transactionalEntityManager.create(this.options.dialect.writeCheckpointEntity, { + id: uuid.v4(), + syncRulesId: null, userId: checkpoint.user_id, - syncRulesId: null - }, - { - orderBy: { checkpoint: 'DESC' }, - limit: 1 - } - ); - createdCheckpoint = (latest?.checkpoint ?? 0n) + 1n; - const row = transactionalEntityManager.create(this.options.dialect.writeCheckpointEntity, { - id: uuid.v4(), - syncRulesId: null, - userId: checkpoint.user_id, - checkpoint: createdCheckpoint, - heads: checkpoint.heads, - createdAt: new Date() - }); - - transactionalEntityManager.persist(row); + checkpoint: value, + heads: checkpoint.heads, + createdAt: new Date() + }); + transactionalEntityManager.persist(row); + createdCheckpoints.set(checkpoint.user_id, value); + } await transactionalEntityManager.flush(); }); this.factory.checkpointWatcher.notify(); - return createdCheckpoint!; + return createdCheckpoints; } async lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { @@ -158,16 +164,17 @@ export class MikroOrmSyncRulesStorage return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; } - getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig { - const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {}; - if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) { - this.parsedSyncRulesCache = { - parsed: this.options.replicationStream.parsed(options).hydratedSyncConfig(), - options - }; + getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet { + let parsed = this.parsedSyncConfigSets.get(options.defaultSchema); + if (parsed == null) { + parsed = this.options.replicationStream.parsed(options); + this.parsedSyncConfigSets.set(options.defaultSchema, parsed); } + return parsed; + } - return this.parsedSyncRulesCache!.parsed; + getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig { + return this.getParsedSyncConfigSet(options).hydratedSyncConfig; } async terminate(options?: storage.TerminateOptions): Promise { @@ -187,18 +194,18 @@ export class MikroOrmSyncRulesStorage this.factory.checkpointWatcher.notify(); } - async getStatus(): Promise { + async getStatus(): Promise { const em = this.options.orm.em.fork(); const row = await em.findOne(this.options.dialect.syncRulesEntity, { id: this.replicationStreamId }); + if (row == null) { + throw new Error('Cannot find replication stream status'); + } return { - checkpoint_lsn: row?.lastCheckpointLsn ?? null, - active: row?.state == storage.SyncRuleState.ACTIVE, - snapshot_done: row?.snapshotDone ?? false, - snapshot_lsn: row?.snapshotLsn ?? null, - keepalive_op: row?.keepaliveOp ?? null + snapshotDone: row.snapshotDone && row.lastCheckpointLsn != null, + resumeLsn: utils.maxLsn(row.snapshotLsn, row.lastCheckpointLsn) }; } @@ -343,7 +350,7 @@ export class MikroOrmSyncRulesStorage } async *getBucketDataBatch( - checkpoint: bigint, + checkpoint: ReplicationCheckpoint, dataBuckets: BucketDataRequest[], options?: BucketDataBatchOptions ): AsyncIterable { @@ -357,7 +364,7 @@ export class MikroOrmSyncRulesStorage const rows = this.options.dialect.streamBucketDataRows({ em: this.options.orm.em.fork(), groupId: this.replicationStreamId, - checkpoint, + checkpoint: checkpoint.checkpoint, dataBuckets, limit: batchRowLimit }); @@ -425,7 +432,7 @@ export class MikroOrmSyncRulesStorage } } - async getChecksums(checkpoint: bigint, buckets: BucketChecksumRequest[]): Promise { + async getChecksums(checkpoint: ReplicationCheckpoint, buckets: BucketChecksumRequest[]): Promise { const result: utils.ChecksumMap = new Map(); for (const bucket of buckets) { const rows = await this.options.orm.em.fork().find( @@ -433,7 +440,7 @@ export class MikroOrmSyncRulesStorage { groupId: this.replicationStreamId, bucketName: bucket.bucket, - opId: { $lte: checkpoint } + opId: { $lte: checkpoint.checkpoint } }, { orderBy: { opId: 'ASC' } diff --git a/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts b/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts index 3ba344c5f..f85bdcce1 100644 --- a/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts +++ b/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts @@ -20,14 +20,12 @@ bucket_definitions: const stream = await factory.updateSyncRules(syncRules); const bucketStorage = factory.getInstance(stream); - const first = await bucketStorage.createManagedWriteCheckpoint({ - user_id: 'user1', - heads: { '1': '5/0' } - }); - const second = await bucketStorage.createManagedWriteCheckpoint({ - user_id: 'user1', - heads: { '1': '6/0' } - }); + const first = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '5/0' } }]) + ).get('user1')!; + const second = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '6/0' } }]) + ).get('user1')!; await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '4/0' } })).resolves.toBeNull(); await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '5/0' } })).resolves.toBe(first); @@ -45,10 +43,9 @@ bucket_definitions: .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) [Symbol.asyncIterator](); - const writeCheckpoint = await bucketStorage.createManagedWriteCheckpoint({ - user_id: 'user1', - heads: { '1': '5/0' } - }); + const writeCheckpoint = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '5/0' } }]) + ).get('user1')!; const em = factory.orm.em.fork(); const row = await em.findOneOrFail(factory.dialect.syncRulesEntity, { From bbe958cb8ed0d01e12f3947dcc071ad4bcdc53d0 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 20 Jul 2026 13:52:34 +0200 Subject: [PATCH 10/11] wip compare benchmarks --- package.json | 1 + .../benchmarks/register-storage-benchmarks.ts | 155 ++++-- plans/sync-bucket-storage-benchmarks.md | 55 ++- scripts/compare-storage-benchmarks.mts | 451 ++++++++++++++++++ 4 files changed, 608 insertions(+), 54 deletions(-) create mode 100644 scripts/compare-storage-benchmarks.mts diff --git a/package.json b/package.json index 7c26758d9..31e4dd25f 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "author": "PowerSync", "scripts": { "check-updates": "ncu -u --deep", + "benchmark:storage": "node scripts/compare-storage-benchmarks.mts", "validate:tsconfig-references": "node scripts/validate-tsconfig-references.mts", "format": "prettier --write .", "format:check": "prettier --check .", diff --git a/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts b/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts index fe6f71df0..f585cc7ac 100644 --- a/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts +++ b/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts @@ -1,5 +1,7 @@ import { createCoreAPIMetrics, JwtPayload, storage, sync, updateSyncRulesFromYaml } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; import { performance } from 'node:perf_hooks'; import { afterAll, test } from 'vitest'; import { @@ -57,41 +59,63 @@ export interface StorageBenchmarkResult { buckets: number; write_ms: number; write_rows_per_second: number; + write_mebibytes_per_second: number; sync_drain_ms: number; ops: number; - mebibytes_per_second: number; + read_mebibytes_per_second: number; } -export const DEFAULT_STORAGE_BENCHMARK_SCENARIOS: StorageBenchmarkScenario[] = [ +export interface StorageBenchmarkOutput { + schema_version: 1; + generated_at: string; + results: StorageBenchmarkResult[]; +} + +export const STORAGE_BENCHMARK_OUTPUT_PATH_ENV = 'POWERSYNC_STORAGE_BENCHMARK_OUTPUT'; + +const STORAGE_BENCHMARK_ROW_PERMUTATIONS = [ { - name: 'small', + name: '1k-todos', todo_row_count: 1_000, - list_row_count: 100, - user_count: 100, - flush_every: 1_000, - max_bucket_count: 1_000, - timeout_ms: 120_000 + timeout_ms: 300_000 }, { - name: 'medium', + name: '10k-todos', todo_row_count: 10_000, - list_row_count: 500, - user_count: 500, - flush_every: 1_000, - max_bucket_count: 1_000, - timeout_ms: 300_000 + timeout_ms: 600_000 }, { - name: 'large', + name: '100k-todos', todo_row_count: 100_000, - list_row_count: 500, - user_count: 500, - flush_every: 1_000, - max_bucket_count: 1_000, - timeout_ms: 1_200_000 + timeout_ms: 1_800_000 + }, + { + name: '1m-todos', + todo_row_count: 1_000_000, + timeout_ms: 10_800_000 } ]; +const STORAGE_BENCHMARK_BUCKET_PERMUTATIONS = [ + { name: '200-buckets', list_row_count: 100, bucket_count: 200, minimum_timeout_ms: 0 }, + { name: '1k-buckets', list_row_count: 500, bucket_count: 1_000, minimum_timeout_ms: 0 }, + { name: '10k-buckets', list_row_count: 5_000, bucket_count: 10_000, minimum_timeout_ms: 900_000 }, + { name: '20k-buckets', list_row_count: 10_000, bucket_count: 20_000, minimum_timeout_ms: 1_200_000 } +]; + +export const DEFAULT_STORAGE_BENCHMARK_SCENARIOS: StorageBenchmarkScenario[] = + STORAGE_BENCHMARK_ROW_PERMUTATIONS.flatMap((rowPermutation) => + STORAGE_BENCHMARK_BUCKET_PERMUTATIONS.map((bucketPermutation) => ({ + name: `${rowPermutation.name}-${bucketPermutation.name}`, + todo_row_count: rowPermutation.todo_row_count, + list_row_count: bucketPermutation.list_row_count, + user_count: bucketPermutation.list_row_count, + flush_every: 1_000, + max_bucket_count: bucketPermutation.bucket_count, + timeout_ms: Math.max(rowPermutation.timeout_ms, bucketPermutation.minimum_timeout_ms) + })) + ); + export function registerStorageBenchmarks( configOrFactory: storage.TestStorageConfig | storage.TestStorageFactory, options: RegisterStorageBenchmarkOptions @@ -128,19 +152,35 @@ export function registerStorageBenchmarkSummary( results: StorageBenchmarkResult[], title = 'Storage benchmark results' ) { - afterAll(() => { + afterAll(async () => { if (results.length == 0) { return; } console.log(`\n${title}\n${formatStorageBenchmarkResults(results)}`); + + const outputPath = process.env[STORAGE_BENCHMARK_OUTPUT_PATH_ENV]; + if (outputPath != null && outputPath.length > 0) { + await writeStorageBenchmarkResults(outputPath, results); + console.log(`\nWrote storage benchmark results to ${outputPath}`); + } }); } +export async function writeStorageBenchmarkResults(outputPath: string, results: StorageBenchmarkResult[]) { + const output: StorageBenchmarkOutput = { + schema_version: 1, + generated_at: new Date().toISOString(), + results + }; + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(output, null, 2)}\n`, 'utf8'); +} + export function formatStorageBenchmarkResults(results: StorageBenchmarkResult[]) { const rows = [ - '| Storage | Version | Scenario | Source Rows | Buckets | Write ms | Write rows/s | Sync drain ms | Ops | MiB/s |', - '| --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |' + '| Storage | Version | Scenario | Source Rows | Buckets | Write ms | Write rows/s | Write MiB/s | Sync drain ms | Ops | Read MiB/s |', + '| --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |' ]; for (const result of results) { @@ -148,8 +188,8 @@ export function formatStorageBenchmarkResults(results: StorageBenchmarkResult[]) `| ${result.storage} | ${result.version ?? ''} | ${result.scenario} | ${result.source_rows} | ${ result.buckets } | ${formatNumber(result.write_ms)} | ${formatNumber(result.write_rows_per_second)} | ${formatNumber( - result.sync_drain_ms - )} | ${result.ops} | ${formatNumber(result.mebibytes_per_second)} |` + result.write_mebibytes_per_second + )} | ${formatNumber(result.sync_drain_ms)} | ${result.ops} | ${formatNumber(result.read_mebibytes_per_second)} |` ); } @@ -175,6 +215,7 @@ async function runStorageBenchmark( const bucketStorage = factory.getInstance(syncRules.stream); const label = benchmarkLabel(options.storageName, options.storageVersion, scenario.name); const sourceRows = scenario.todo_row_count + scenario.list_row_count; + const sourceBytes = calculateSourceBytes(scenario); const writeStarted = performance.now(); await persistBenchmarkRows({ @@ -201,9 +242,10 @@ async function runStorageBenchmark( buckets: drain.buckets, write_ms: writeMs, write_rows_per_second: sourceRows / (writeMs / 1_000), + write_mebibytes_per_second: sourceBytes / 1024 / 1024 / (writeMs / 1_000), sync_drain_ms: drain.elapsedMs, ops: drain.ops, - mebibytes_per_second: drain.bytes / 1024 / 1024 / (drain.elapsedMs / 1_000) + read_mebibytes_per_second: drain.bytes / 1024 / 1024 / (drain.elapsedMs / 1_000) }; } @@ -254,28 +296,17 @@ async function persistBenchmarkRows(options: { await saveRow({ sourceTable: listsTable, tag: storage.SaveOperationTag.INSERT, - after: { - id: listId, - owner_id: userId(i % scenario.user_count), - name: `List ${i}` - }, + after: listRow(i, scenario), afterReplicaId: listId }); } for (let i = 0; i < scenario.todo_row_count; i++) { - const listIndex = i % scenario.list_row_count; const todoId = todoRowId(i); await saveRow({ sourceTable: todosTable, tag: storage.SaveOperationTag.INSERT, - after: { - id: todoId, - list_id: listRowId(listIndex), - owner_id: userId(listIndex % scenario.user_count), - description: `Todo ${i} for ${listRowId(listIndex)}`, - completed: i % 3 == 0 ? 1 : 0 - }, + after: todoRow(i, scenario), afterReplicaId: todoId }); } @@ -294,7 +325,7 @@ async function drainSyncStream(options: { const started = performance.now(); const tracker = new sync.RequestTracker(METRICS_HELPER.metricsEngine); const controller = new AbortController(); - const maxBucketCount = expectedBucketCount(scenario); + const maxBucketCount = configuredMaxBucketCount(scenario); const syncContext = new sync.SyncContext({ maxBuckets: maxBucketCount + 10, maxParameterQueryResults: maxBucketCount + 10, @@ -395,7 +426,7 @@ function validateScenario(scenario: StorageBenchmarkScenario) { throw new Error(`Benchmark scenario ${scenario.name} must flush at least every one row`); } - const maxBucketCount = scenario.max_bucket_count ?? 1_000; + const maxBucketCount = configuredMaxBucketCount(scenario); const buckets = expectedBucketCount(scenario); if (buckets > maxBucketCount) { throw new Error( @@ -408,6 +439,40 @@ function expectedBucketCount(scenario: StorageBenchmarkScenario) { return scenario.list_row_count + Math.min(scenario.list_row_count, scenario.user_count); } +function configuredMaxBucketCount(scenario: StorageBenchmarkScenario) { + return scenario.max_bucket_count ?? 1_000; +} + +function calculateSourceBytes(scenario: StorageBenchmarkScenario) { + let bytes = 0; + for (let i = 0; i < scenario.list_row_count; i++) { + bytes += Buffer.byteLength(JSONBig.stringify(listRow(i, scenario))); + } + for (let i = 0; i < scenario.todo_row_count; i++) { + bytes += Buffer.byteLength(JSONBig.stringify(todoRow(i, scenario))); + } + return bytes; +} + +function listRow(index: number, scenario: StorageBenchmarkScenario) { + return { + id: listRowId(index), + owner_id: userId(index % scenario.user_count), + name: `List ${index}` + }; +} + +function todoRow(index: number, scenario: StorageBenchmarkScenario) { + const listIndex = index % scenario.list_row_count; + return { + id: todoRowId(index), + list_id: listRowId(listIndex), + owner_id: userId(listIndex % scenario.user_count), + description: `Todo ${index} for ${listRowId(listIndex)}`, + completed: index % 3 == 0 ? 1 : 0 + }; +} + function benchmarkLabel(storageName: string, storageVersion: number | undefined, scenarioName: string) { return `${storageName}/v${storageVersion ?? 'default'}/${scenarioName}`; } @@ -427,9 +492,11 @@ function userId(index: number) { function formatStorageBenchmarkResultLine(result: StorageBenchmarkResult) { return `[${result.storage}/v${result.version ?? 'default'}/${result.scenario}] result write_ms=${formatNumber( result.write_ms - )} write_rows_per_second=${formatNumber(result.write_rows_per_second)} sync_drain_ms=${formatNumber( - result.sync_drain_ms - )} ops=${result.ops} buckets=${result.buckets} mebibytes_per_second=${formatNumber(result.mebibytes_per_second)}`; + )} write_rows_per_second=${formatNumber(result.write_rows_per_second)} write_mebibytes_per_second=${formatNumber( + result.write_mebibytes_per_second + )} sync_drain_ms=${formatNumber(result.sync_drain_ms)} ops=${result.ops} buckets=${ + result.buckets + } read_mebibytes_per_second=${formatNumber(result.read_mebibytes_per_second)}`; } function formatNumber(value: number) { diff --git a/plans/sync-bucket-storage-benchmarks.md b/plans/sync-bucket-storage-benchmarks.md index cebfefc20..5e7870c92 100644 --- a/plans/sync-bucket-storage-benchmarks.md +++ b/plans/sync-bucket-storage-benchmarks.md @@ -34,15 +34,25 @@ interface StorageBenchmarkScenario { } ``` -Default scenarios: - -| Scenario | Todo Rows | List Rows | Users | Max Buckets | -| -------- | --------: | --------: | ----: | ----------: | -| small | 1,000 | 100 | 100 | 1,000 | -| medium | 10,000 | 500 | 500 | 1,000 | -| large | 100,000 | 500 | 500 | 1,000 | - -The number of rows in buckets may grow large, but the total resolved bucket count must stay capped at 1,000 by default. All registered scenarios always run both the write phase and the sync stream drain phase. +Default scenarios are the cross-product of these dimensions: + +| Todo Rows | Timeout | +| --------: | ------: | +| 1,000 | 5 min | +| 10,000 | 10 min | +| 100,000 | 30 min | +| 1,000,000 | 3 h | + +| Resolved Buckets | List Rows | Users | Minimum Timeout | +| ---------------: | --------: | -----: | --------------: | +| 200 | 100 | 100 | - | +| 1,000 | 500 | 500 | - | +| 10,000 | 5,000 | 5,000 | 15 min | +| 20,000 | 10,000 | 10,000 | 20 min | + +This produces 16 scenarios. Every row-count permutation includes the original 200- and 1,000-bucket shapes plus the +new 10,000- and 20,000-bucket shapes. Each scenario passes its own `max_bucket_count` into the sync context for both +bucket and parameter-result limits. All registered scenarios run both the write phase and sync stream drain phase. ## Harness Requirements @@ -105,8 +115,33 @@ The summary table should include: - Buckets - Write ms - Write rows/s +- Write MiB/s, based on the logical JSON size of source rows - Sync drain ms - Ops -- MiB/s +- Read MiB/s, based on bytes emitted by the sync stream The output is intentionally plain markdown so benchmark results can be pasted into issues, PRs, or follow-up analysis notes. + +## Comparing Storage Implementations + +Use the root comparison CLI to select and run multiple storage implementations: + +```sh +pnpm benchmark:storage +``` + +Missing storage selections are prompted for interactively. They can also be supplied directly: + +```sh +pnpm benchmark:storage --storage drizzle-sqlite,mikroorm-sqlite +``` + +Use `--output ` to retain the combined JSON results. The CLI runs each selected module in a separate Vitest +process, collects its machine-readable output, and prints throughput relative to the fastest selected implementation +for each scenario. + +The CLI also writes a self-contained grouped SVG chart to `storage-benchmark-comparison.svg`, comparing write and read +MiB/s for every storage/version/scenario result. Use `--chart ` to select a different destination. + +A benchmark test can also write its results directly by setting `POWERSYNC_STORAGE_BENCHMARK_OUTPUT` to the desired +JSON path. Without that environment variable, benchmark tests keep their existing console-only behavior. diff --git a/scripts/compare-storage-benchmarks.mts b/scripts/compare-storage-benchmarks.mts new file mode 100644 index 000000000..e60c42205 --- /dev/null +++ b/scripts/compare-storage-benchmarks.mts @@ -0,0 +1,451 @@ +import inquirer from 'inquirer'; +import { spawn } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const OUTPUT_ENV = 'POWERSYNC_STORAGE_BENCHMARK_OUTPUT'; + +interface StorageBenchmarkResult { + storage: string; + version: number | null; + scenario: string; + source_rows: number; + buckets: number; + write_ms: number; + write_rows_per_second: number; + write_mebibytes_per_second: number; + sync_drain_ms: number; + ops: number; + read_mebibytes_per_second: number; +} + +interface StorageBenchmarkOutput { + schema_version: 1; + generated_at: string; + results: StorageBenchmarkResult[]; +} + +interface StorageRunner { + id: string; + label: string; + packageName: string; + testName?: string; +} + +const STORAGE_RUNNERS: StorageRunner[] = [ + { + id: 'mongodb', + label: 'MongoDB', + packageName: '@powersync/service-module-mongodb-storage' + }, + { + id: 'postgresql', + label: 'PostgreSQL', + packageName: '@powersync/service-module-postgres-storage' + }, + { + id: 'drizzle-sqlite', + label: 'Drizzle SQLite', + packageName: '@powersync/service-module-drizzle-storage' + }, + { + id: 'mikroorm-sqlite', + label: 'MikroORM SQLite', + packageName: '@powersync/service-module-mikroorm-storage', + testName: 'MikroORM SQLite' + }, + { + id: 'mikroorm-mysql', + label: 'MikroORM MySQL', + packageName: '@powersync/service-module-mikroorm-storage', + testName: 'MikroORM MySQL' + } +]; + +const STORAGE_ALIASES = new Map([ + ['mongo', 'mongodb'], + ['postgres', 'postgresql'], + ['drizzle:sqlite', 'drizzle-sqlite'], + ['mikroorm:sqlite', 'mikroorm-sqlite'], + ['mikroorm:mysql', 'mikroorm-mysql'] +]); + +interface CliOptions { + storageIds: string[]; + outputPath?: string; + chartPath: string; + help: boolean; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + printHelp(); + return; + } + + const selectedIds = options.storageIds.length > 0 ? options.storageIds : await promptForStorageIds(); + const runners = selectRunners(selectedIds); + const temporaryDirectory = await mkdtemp(join(tmpdir(), 'powersync-storage-benchmarks-')); + const results: StorageBenchmarkResult[] = []; + const failures: string[] = []; + + try { + for (const runner of runners) { + const resultPath = join(temporaryDirectory, `${runner.id}.json`); + console.log(`\nRunning ${runner.label} storage benchmarks...\n`); + + const exitCode = await runStorageBenchmark(runner, resultPath); + if (exitCode != 0) { + failures.push(`${runner.label} (exit code ${exitCode})`); + continue; + } + + try { + const output = parseBenchmarkOutput(await readFile(resultPath, 'utf8'), resultPath); + results.push(...output.results); + } catch (error) { + failures.push(`${runner.label} (${error instanceof Error ? error.message : String(error)})`); + } + } + + if (results.length > 0) { + console.log(`\nStorage benchmark comparison\n\n${formatComparison(results)}`); + await writeComparisonChart(options.chartPath, results); + console.log(`\nWrote benchmark comparison chart to ${options.chartPath}`); + } + + if (options.outputPath != null) { + const output: StorageBenchmarkOutput = { + schema_version: 1, + generated_at: new Date().toISOString(), + results + }; + await mkdir(dirname(options.outputPath), { recursive: true }); + await writeFile(options.outputPath, `${JSON.stringify(output, null, 2)}\n`, 'utf8'); + console.log(`\nWrote combined benchmark results to ${options.outputPath}`); + } + + if (failures.length > 0) { + console.error(`\nFailed benchmark runs:\n${failures.map((failure) => `- ${failure}`).join('\n')}`); + process.exitCode = 1; + } + } finally { + await rm(temporaryDirectory, { recursive: true, force: true }); + } +} + +function parseArgs(args: string[]): CliOptions { + const storageIds: string[] = []; + let outputPath: string | undefined; + let chartPath = 'storage-benchmark-comparison.svg'; + let help = false; + + for (let index = 0; index < args.length; index++) { + const argument = args[index]; + if (argument == '--help' || argument == '-h') { + help = true; + } else if (argument == '--storage' || argument == '-s') { + storageIds.push(...readOptionValue(args, ++index, argument).split(',')); + } else if (argument.startsWith('--storage=')) { + storageIds.push(...argument.slice('--storage='.length).split(',')); + } else if (argument == '--output' || argument == '-o') { + outputPath = readOptionValue(args, ++index, argument); + } else if (argument.startsWith('--output=')) { + outputPath = argument.slice('--output='.length); + } else if (argument == '--chart' || argument == '-c') { + chartPath = readOptionValue(args, ++index, argument); + } else if (argument.startsWith('--chart=')) { + chartPath = argument.slice('--chart='.length); + } else { + throw new Error(`Unknown option: ${argument}`); + } + } + + return { + storageIds: storageIds.map((id) => id.trim()).filter(Boolean), + outputPath, + chartPath, + help + }; +} + +function readOptionValue(args: string[], index: number, option: string) { + const value = args[index]; + if (value == null || value.startsWith('-')) { + throw new Error(`Missing value for ${option}`); + } + return value; +} + +async function promptForStorageIds() { + const answer = await inquirer.prompt<{ storageIds: string[] }>([ + { + type: 'checkbox', + name: 'storageIds', + message: 'Which bucket storage implementations should be compared?', + choices: STORAGE_RUNNERS.map((runner) => ({ + name: runner.label, + value: runner.id, + checked: runner.id == 'drizzle-sqlite' || runner.id == 'mikroorm-sqlite' + })), + validate: (selected: string[]) => selected.length > 0 || 'Select at least one storage implementation' + } + ]); + return answer.storageIds; +} + +function selectRunners(storageIds: string[]) { + const normalizedIds = storageIds.flatMap((id) => { + if (id == 'all') { + return STORAGE_RUNNERS.map((runner) => runner.id); + } + return [STORAGE_ALIASES.get(id) ?? id]; + }); + const unknownIds = normalizedIds.filter((id) => !STORAGE_RUNNERS.some((runner) => runner.id == id)); + if (unknownIds.length > 0) { + throw new Error(`Unknown storage implementation(s): ${[...new Set(unknownIds)].join(', ')}`); + } + + return [...new Set(normalizedIds)].map((id) => STORAGE_RUNNERS.find((runner) => runner.id == id)!); +} + +function runStorageBenchmark(runner: StorageRunner, resultPath: string) { + const command = process.platform == 'win32' ? 'corepack.cmd' : 'corepack'; + const args = ['pnpm', '--filter', runner.packageName, 'test', 'test/src/storage_bench.test.ts', '--run']; + if (runner.testName != null) { + args.push('-t', runner.testName); + } + + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: fileURLToPath(new URL('..', import.meta.url)), + env: { + ...process.env, + [OUTPUT_ENV]: resultPath + }, + stdio: 'inherit' + }); + child.once('error', reject); + child.once('close', (code, signal) => { + if (signal != null) { + console.error(`${runner.label} benchmark terminated by ${signal}`); + } + resolve(code ?? 1); + }); + }); +} + +function parseBenchmarkOutput(json: string, source: string): StorageBenchmarkOutput { + const parsed = JSON.parse(json) as Partial; + if (parsed.schema_version != 1 || !Array.isArray(parsed.results)) { + throw new Error(`Invalid benchmark output in ${source}`); + } + return parsed as StorageBenchmarkOutput; +} + +function formatComparison(results: StorageBenchmarkResult[]) { + const fastestWrites = maximumByScenario(results, (result) => result.write_mebibytes_per_second); + const fastestDrains = maximumByScenario(results, (result) => result.read_mebibytes_per_second); + const scenarioOrder = new Map(); + for (const result of results) { + if (!scenarioOrder.has(result.scenario)) { + scenarioOrder.set(result.scenario, scenarioOrder.size); + } + } + const rows = [ + '| Scenario | Storage | Version | Write rows/s | Write MiB/s | Write relative | Read MiB/s | Read relative | Write ms | Sync drain ms |', + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |' + ]; + + for (const result of [...results].sort((left, right) => compareResults(left, right, scenarioOrder))) { + rows.push( + `| ${result.scenario} | ${result.storage} | ${result.version ?? ''} | ${formatNumber( + result.write_rows_per_second + )} | ${formatNumber(result.write_mebibytes_per_second)} | ${formatPercent( + result.write_mebibytes_per_second, + fastestWrites.get(result.scenario)! + )} | ${formatNumber(result.read_mebibytes_per_second)} | ${formatPercent( + result.read_mebibytes_per_second, + fastestDrains.get(result.scenario)! + )} | ${formatNumber(result.write_ms)} | ${formatNumber(result.sync_drain_ms)} |` + ); + } + return rows.join('\n'); +} + +function maximumByScenario(results: StorageBenchmarkResult[], value: (result: StorageBenchmarkResult) => number) { + const maximums = new Map(); + for (const result of results) { + maximums.set(result.scenario, Math.max(maximums.get(result.scenario) ?? 0, value(result))); + } + return maximums; +} + +function compareResults( + left: StorageBenchmarkResult, + right: StorageBenchmarkResult, + scenarioOrder: Map +) { + return ( + scenarioOrder.get(left.scenario)! - scenarioOrder.get(right.scenario)! || left.storage.localeCompare(right.storage) + ); +} + +function formatPercent(value: number, maximum: number) { + return maximum == 0 ? 'n/a' : `${((value / maximum) * 100).toFixed(1)}%`; +} + +function formatNumber(value: number) { + if (Number.isInteger(value)) { + return String(value); + } + return value.toFixed(value >= 100 ? 0 : 2); +} + +export async function writeComparisonChart(outputPath: string, results: StorageBenchmarkResult[]) { + const width = Math.max(960, results.length * 110 + 180); + const height = 620; + const margin = { top: 90, right: 40, bottom: 190, left: 90 }; + const plotWidth = width - margin.left - margin.right; + const plotHeight = height - margin.top - margin.bottom; + const maximum = niceCeiling( + Math.max(...results.flatMap((result) => [result.write_mebibytes_per_second, result.read_mebibytes_per_second])) + ); + const scenarioOrder = new Map(); + for (const result of results) { + if (!scenarioOrder.has(result.scenario)) { + scenarioOrder.set(result.scenario, scenarioOrder.size); + } + } + const orderedResults = [...results].sort((left, right) => compareResults(left, right, scenarioOrder)); + const groupWidth = plotWidth / orderedResults.length; + const barWidth = Math.min(30, Math.max(8, groupWidth * 0.32)); + const barGap = Math.min(8, groupWidth * 0.08); + const baseline = margin.top + plotHeight; + const elements: string[] = []; + + for (let tick = 0; tick <= 5; tick++) { + const value = (maximum * tick) / 5; + const y = baseline - (plotHeight * tick) / 5; + elements.push( + ``, + `${escapeXml( + formatNumber(value) + )}` + ); + } + + orderedResults.forEach((result, index) => { + const center = margin.left + groupWidth * (index + 0.5); + const writeHeight = (result.write_mebibytes_per_second / maximum) * plotHeight; + const readHeight = (result.read_mebibytes_per_second / maximum) * plotHeight; + const writeX = center - barGap / 2 - barWidth; + const readX = center + barGap / 2; + const label = `${result.storage} v${result.version ?? 'default'} ยท ${result.scenario}`; + elements.push( + chartBar(writeX, baseline - writeHeight, barWidth, writeHeight, '#2563eb', 'Write', result, label), + chartBar(readX, baseline - readHeight, barWidth, readHeight, '#f97316', 'Read', result, label), + `${escapeXml(label)}` + ); + }); + + const svg = ` + + Storage benchmark write and read throughput + Grouped bar chart comparing logical source write throughput and sync stream read throughput in mebibytes per second. + + + Storage benchmark throughput + Logical source writes vs. sync stream reads (MiB/s) + + Write + + Read + MiB/s + ${elements.join('\n ')} + + +`; + + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, svg, 'utf8'); +} + +function chartBar( + x: number, + y: number, + width: number, + height: number, + color: string, + metric: string, + result: StorageBenchmarkResult, + label: string +) { + const value = metric == 'Write' ? result.write_mebibytes_per_second : result.read_mebibytes_per_second; + return `${escapeXml( + `${label}: ${metric} ${formatNumber(value)} MiB/s` + )}`; +} + +function niceCeiling(value: number) { + if (!Number.isFinite(value) || value <= 0) { + return 1; + } + const magnitude = 10 ** Math.floor(Math.log10(value)); + const normalized = value / magnitude; + const nice = normalized <= 1 ? 1 : normalized <= 2 ? 2 : normalized <= 5 ? 5 : 10; + return nice * magnitude; +} + +function escapeXml(value: string) { + return value.replace(/[&<>"']/g, (character) => { + return { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''' + }[character]!; + }); +} + +function printHelp() { + console.log(`Compare sync bucket storage benchmarks. + +Usage: + pnpm benchmark:storage [options] + +Options: + -s, --storage Comma-separated storage IDs; repeatable. Use "all" for every implementation. + -o, --output Write the combined JSON results to this path. + -c, --chart Write the SVG chart to this path (default: storage-benchmark-comparison.svg). + -h, --help Show this help. + +Storage IDs: +${STORAGE_RUNNERS.map((runner) => ` ${runner.id.padEnd(18)} ${runner.label}`).join('\n')} + +Database-backed runners use their module's normal test connection environment variables.`); +} + +if (process.argv[1] != null && import.meta.url == pathToFileURL(resolve(process.argv[1])).href) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} From 40c9fce3856079ec8752d10df236bf424405532e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 21 Jul 2026 09:28:26 +0200 Subject: [PATCH 11/11] wip: postgres performance improvements --- .../storage/PostgresBucketStorageFactory.ts | 4 +- .../src/storage/PostgresSyncRulesStorage.ts | 111 +++++++++--------- .../src/utils/test-utils.ts | 9 +- .../test/src/storage.test.ts | 53 ++++++++- .../test/src/storage_bench.test.ts | 9 +- .../module-postgres-storage/test/src/util.ts | 28 ++++- 6 files changed, 154 insertions(+), 60 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 79fb9519c..40860c4af 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -16,6 +16,7 @@ export type PostgresBucketStorageOptions = { config: NormalizedPostgresStorageConfig; replicationStreamNamePrefix: string; checksumCacheTtlMs?: number; + bucketDataQueryHook?: (db: lib_postgres.DatabaseClient, query: pg_wire.Statement) => Promise; }; export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { @@ -58,7 +59,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { db: this.db, replicationStream, batchLimits: this.options.config.batch_limits, - checksumCacheTtlMs: this.options.checksumCacheTtlMs + checksumCacheTtlMs: this.options.checksumCacheTtlMs, + bucketDataQueryHook: this.options.bucketDataQueryHook }); if (!options?.skipLifecycleHooks) { this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage)); diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 85afdfdf5..36cd933c0 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -28,7 +28,7 @@ import { replicaIdToSubkey } from '../utils/bson.js'; import { mapOpEntry } from '../utils/bucket-data.js'; import * as framework from '@powersync/lib-services-framework'; -import { StatementParam } from '@powersync/service-jpgwire'; +import type { Statement } from '@powersync/service-jpgwire'; import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js'; import * as t from 'ts-codec'; import { pick } from '../utils/ts-codec.js'; @@ -45,6 +45,7 @@ export type PostgresSyncRulesStorageOptions = { write_checkpoint_mode?: storage.WriteCheckpointMode; batchLimits: RequiredOperationBatchLimits; checksumCacheTtlMs?: number; + bucketDataQueryHook?: (db: lib_postgres.DatabaseClient, query: Statement) => Promise; }; export class PostgresSyncRulesStorage @@ -327,7 +328,7 @@ export class PostgresSyncRulesStorage // not match up with chunks. const end = checkpoint.checkpoint ?? BIGINT_MAX; - const filters = dataBuckets.map((request) => ({ bucket_name: request.bucket, start: request.start })); + const sortedBuckets = [...dataBuckets].sort((a, b) => a.bucket.localeCompare(b.bucket)); const startOpByBucket = new Map(dataBuckets.map((request) => [request.bucket, request.start])); const batchRowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; @@ -338,65 +339,67 @@ export class PostgresSyncRulesStorage let targetOp: InternalOpId | null = null; let batchRowCount = 0; - /** - * It is possible to perform this query with JSONB join. e.g. - * ```sql - * WITH - * filter_data AS ( - * SELECT - * FILTER ->> 'bucket_name' AS bucket_name, - * (FILTER ->> 'start')::BIGINT AS start_op_id - * FROM - * jsonb_array_elements($1::jsonb) AS FILTER - * ) - * SELECT - * b.*, - * octet_length(b.data) AS data_size - * FROM - * bucket_data b - * JOIN filter_data f ON b.bucket_name = f.bucket_name - * AND b.op_id > f.start_op_id - * AND b.op_id <= $2 - * WHERE - * b.group_id = $3 - * ORDER BY - * b.bucket_name ASC, - * b.op_id ASC - * LIMIT - * $4; - * ``` - * Which might be better for large volumes of buckets, but in testing the JSON method - * was significantly slower than the method below. Syncing 2.5 million rows in a single - * bucket takes 2 minutes and 11 seconds with the method below. With the JSON method - * 1 million rows were only synced before a 5 minute timeout. - */ - for await (const rows of this.db.streamRows({ + const requestedBuckets = sortedBuckets + .map((_, index) => `($${index * 2 + 4}, $${index * 2 + 5}, ${index})`) + .join(', '); + + // Use one round trip while keeping a parameterized index range scan per bucket. Ordering the non-flattened request + // subquery gives Postgres a presorted bucket key, allowing an incremental sort and the outer limit to stop the + // nested loop early. The inner limit cannot exclude rows needed by the outer batch because both limits are equal. + const query: Statement = { statement: ` - SELECT - * - FROM - bucket_data - WHERE - group_id = $1 - and op_id <= $2 - and ( - ${filters.map((f, index) => `(bucket_name = $${index * 2 + 4} and op_id > $${index * 2 + 5})`).join(' OR ')} - ) - ORDER BY - bucket_name ASC, - op_id ASC - LIMIT - $3;`, + SELECT + bucket_data.* + FROM + ( + SELECT + * + FROM + ( + VALUES ${requestedBuckets} + ) AS bucket_requests(bucket_name, start_op_id, bucket_order) + -- Present requests in bucket order so the final sort can be incremental. + ORDER BY + bucket_order ASC + -- Prevent the planner from flattening away the ordered request path. + OFFSET + 0 + ) AS requested + CROSS JOIN LATERAL ( + SELECT + * + FROM + bucket_data + WHERE + group_id = $1 + AND bucket_name = requested.bucket_name + AND op_id > requested.start_op_id + AND op_id <= $2 + -- Keep each bucket's index range scan ordered and bounded. + ORDER BY + op_id ASC + LIMIT + $3 + ) AS bucket_data + -- Guarantee contiguous bucket chunks while allowing the presorted bucket key to stop scans early. + ORDER BY + requested.bucket_order ASC, + bucket_data.op_id ASC + LIMIT + $3;`, params: [ { type: 'int4', value: this.replicationStreamId }, { type: 'int8', value: end }, { type: 'int4', value: batchRowLimit }, - ...filters.flatMap((f) => [ - { type: 'varchar' as const, value: f.bucket_name }, - { type: 'int8' as const, value: f.start } satisfies StatementParam + ...sortedBuckets.flatMap((request) => [ + { type: 'varchar' as const, value: request.bucket }, + { type: 'int8' as const, value: request.start } ]) ] - })) { + }; + await this.options.bucketDataQueryHook?.(this.db, query); + + for await (const rows of this.db.streamRows(query)) { const decodedRows = rows.map((r) => models.BucketData.decode(r as any)); for (const row of decodedRows) { diff --git a/modules/module-postgres-storage/src/utils/test-utils.ts b/modules/module-postgres-storage/src/utils/test-utils.ts index 6f044600a..5fbea6a57 100644 --- a/modules/module-postgres-storage/src/utils/test-utils.ts +++ b/modules/module-postgres-storage/src/utils/test-utils.ts @@ -1,7 +1,10 @@ import { createLogger, logger as defaultLogger, transports } from '@powersync/lib-services-framework'; import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core'; import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; -import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js'; +import { + PostgresBucketStorageFactory, + type PostgresBucketStorageOptions +} from '../storage/PostgresBucketStorageFactory.js'; import { PostgresReportStorage } from '../storage/PostgresReportStorage.js'; import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js'; import { truncateTables } from './db.js'; @@ -13,6 +16,7 @@ export type PostgresTestStorageOptions = { * This allows for providing a custom PostgresMigrationAgent. */ migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent; + bucketDataQueryHook?: PostgresBucketStorageOptions['bucketDataQueryHook']; }; export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) { @@ -102,7 +106,8 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) { return new PostgresBucketStorageFactory({ config: TEST_CONNECTION_OPTIONS, - replicationStreamNamePrefix: 'test_' + replicationStreamNamePrefix: 'test_', + bucketDataQueryHook: factoryOptions.bucketDataQueryHook }); } catch (ex) { // Vitest does not display these errors nicely when using the `await using` syntx diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index 5dd96ae74..061c2c505 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -1,6 +1,6 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; describe('Sync Bucket Validation', register.registerBucketValidationTests); @@ -20,6 +20,57 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion })); describe(`Postgres Sync Bucket Storage - pg-specific - v${storageVersion}`, () => { + test('queries bucket data with lateral range scans', async () => { + await using factory = await POSTGRES_STORAGE_FACTORY.factory(); + const syncRules = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +bucket_definitions: + alpha: + data: + - SELECT id, bucket FROM test WHERE bucket = 'alpha' + zeta: + data: + - SELECT id, bucket FROM test WHERE bucket = 'zeta' +`, + { storageVersion } + ) + ); + const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = syncRules.syncConfigContent[0]; + const alphaBucket = bucketRequest(syncRulesContent, 'alpha[]'); + const zetaBucket = bucketRequest(syncRulesContent, 'zeta[]'); + + const result = await (async () => { + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY); + + for (const bucket of ['alpha', 'zeta']) { + await writer.save({ + sourceTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: bucket, bucket }, + afterReplicaId: test_utils.rid(bucket) + }); + } + + return writer.flush(); + })(); + + const streamRows = vi.spyOn(factory.db, 'streamRows'); + const batch = await test_utils.fromAsync( + bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(result!.flushed_op), [zetaBucket, alphaBucket]) + ); + + expect(batch.map((chunk) => chunk.chunkData.bucket)).toEqual([alphaBucket.bucket, zetaBucket.bucket]); + expect(streamRows).toHaveBeenCalledTimes(1); + const [query] = streamRows.mock.calls[0]; + expect(query.statement).toContain('CROSS JOIN LATERAL'); + expect(query.statement).toMatch(/ORDER BY\s+bucket_order ASC/); + expect(query.statement).toMatch(/OFFSET\s+0/); + expect([query.params?.[3]?.value, query.params?.[5]?.value]).toEqual([alphaBucket.bucket, zetaBucket.bucket]); + }); + /** * The split of returned results can vary depending on storage drivers. * The large rows here are 2MB large while the default chunk limit is 1mb. diff --git a/modules/module-postgres-storage/test/src/storage_bench.test.ts b/modules/module-postgres-storage/test/src/storage_bench.test.ts index 341f91469..9f68046d2 100644 --- a/modules/module-postgres-storage/test/src/storage_bench.test.ts +++ b/modules/module-postgres-storage/test/src/storage_bench.test.ts @@ -6,14 +6,21 @@ import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; const results: StorageBenchmarkResult[] = []; register.registerStorageBenchmarkSummary(results); +const scenarios = register.DEFAULT_STORAGE_BENCHMARK_SCENARIOS.filter( + (scenario) => + // scenario.todo_row_count < 1_000_000 && + scenario.max_bucket_count == null || scenario.max_bucket_count <= 1_000 +); + describe.sequential('Postgres Sync Bucket Storage Benchmarks', () => { for (const storageVersion of TEST_STORAGE_VERSIONS) { - describe(`v${storageVersion}`, () => { + describe.skipIf(storageVersion !== 1)(`v${storageVersion}`, () => { register.registerStorageBenchmarks( { ...POSTGRES_STORAGE_FACTORY, storageVersion }, { storageName: 'postgresql', storageVersion, + scenarios, results } ); diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts index 88dca61ca..c28b5bb20 100644 --- a/modules/module-postgres-storage/test/src/util.ts +++ b/modules/module-postgres-storage/test/src/util.ts @@ -1,4 +1,6 @@ +import type { DatabaseClient } from '@powersync/lib-service-postgres'; import { LEGACY_STORAGE_VERSION } from '@powersync/service-core'; +import type { Statement } from '@powersync/service-jpgwire'; import path from 'path'; import { fileURLToPath } from 'url'; import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js'; @@ -18,6 +20,29 @@ const BASE_CONFIG = { export const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG); +let explainedBucketDataQuery = false; +let bucketDataQueryCount = 0; + +async function explainBucketDataQuery(db: DatabaseClient, query: Statement) { + bucketDataQueryCount++; + const explainQuery = Number(process.env.POWERSYNC_STORAGE_BENCHMARK_EXPLAIN_QUERY ?? 1); + if ( + process.env.POWERSYNC_STORAGE_BENCHMARK_EXPLAIN !== 'true' || + explainedBucketDataQuery || + bucketDataQueryCount != explainQuery + ) { + return; + } + explainedBucketDataQuery = true; + + const rows = await db.queryRows>({ + statement: `EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT TEXT) ${query.statement}`, + params: query.params + }); + const plan = rows.map((row) => Object.values(row)[0]).join('\n'); + console.log(`\nPostgres bucket data query plan\n${plan}\n`); +} + /** * Vitest tries to load the migrations via .ts files which fails. * For tests this links to the relevant .js files correctly @@ -30,7 +55,8 @@ class TestPostgresMigrationAgent extends PostgresMigrationAgent { export const POSTGRES_STORAGE_SETUP = postgresTestSetup({ url: env.PG_STORAGE_TEST_URL, - migrationAgent: (config) => new TestPostgresMigrationAgent(config) + migrationAgent: (config) => new TestPostgresMigrationAgent(config), + bucketDataQueryHook: explainBucketDataQuery }); export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP;