diff --git a/modules/module-drizzle-storage/README.md b/modules/module-drizzle-storage/README.md new file mode 100644 index 000000000..6a5710878 --- /dev/null +++ b/modules/module-drizzle-storage/README.md @@ -0,0 +1,61 @@ +# Drizzle Bucket Storage + +Experimental Drizzle-backed bucket storage for the PowerSync service. + +The first implementation uses SQLite through `better-sqlite3`: + +```yaml +storage: + type: drizzle:sqlite + filename: ./powersync-storage.sqlite +``` + +SQLite storage is restricted to the unified runner because checkpoint +notifications are process-local. `:memory:` is supported for tests and +throwaway development. + +## Design + +The SQLite driver owns its Drizzle table declarations. Column codecs map +database values to the shared runtime shapes used by storage code, including +lossless `bigint`, `Buffer`, JSON, boolean, and `Date` values. Compile-time +assertions verify that the inferred SQLite query models match the canonical +storage records. + +Storage classes use Drizzle queries directly. Database-specific behavior is +kept behind the small dialect surface only where required: transactions, +bucket reads, compaction/raw SQL, and checkpoint notification. + +## Migrations + +Drizzle Kit owns schema generation and migration metadata: + +```sh +corepack pnpm --filter @powersync/service-module-drizzle-storage drizzle:generate:sqlite +corepack pnpm --filter @powersync/service-module-drizzle-storage drizzle:check:sqlite +``` + +Generated SQL and metadata are committed under `src/migrations/sqlite` and +copied into the built package. Runtime migrations are exposed through the +normal PowerSync migration agent and guarded by a SQLite-backed lock. + +## Development + +```sh +corepack pnpm --filter @powersync/service-module-drizzle-storage build +corepack pnpm --filter @powersync/service-module-drizzle-storage build:tests +corepack pnpm --filter @powersync/service-module-drizzle-storage test --run +``` + +PostgreSQL replication tests can opt into this storage with: + +```sh +TEST_MONGO_STORAGE=false \ +TEST_POSTGRES_STORAGE=false \ +TEST_MIKROORM_SQLITE_STORAGE=false \ +TEST_DRIZZLE_SQLITE_STORAGE=true \ +corepack pnpm --filter @powersync/service-module-postgres test test/src/wal_stream.test.ts --run +``` + +Set `DRIZZLE_SQLITE_STORAGE_TEST_FILENAME` to override the file-backed test +database path. diff --git a/modules/module-drizzle-storage/drizzle.sqlite.config.ts b/modules/module-drizzle-storage/drizzle.sqlite.config.ts new file mode 100644 index 000000000..0b75bb676 --- /dev/null +++ b/modules/module-drizzle-storage/drizzle.sqlite.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'drizzle-kit'; + +export default defineConfig({ + dialect: 'sqlite', + schema: './src/drivers/sqlite/schema.ts', + out: './src/migrations/sqlite' +}); diff --git a/modules/module-drizzle-storage/package.json b/modules/module-drizzle-storage/package.json new file mode 100644 index 000000000..29165aa1a --- /dev/null +++ b/modules/module-drizzle-storage/package.json @@ -0,0 +1,51 @@ +{ + "name": "@powersync/service-module-drizzle-storage", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/index.d.ts", + "version": "0.1.0", + "main": "dist/index.js", + "license": "FSL-1.1-ALv2", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "rm -rf ./dist ./tsconfig.tsbuildinfo && tsc -b && cp -R src/migrations/sqlite dist/migrations/sqlite", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./dist && tsc -b --clean", + "drizzle:generate:sqlite": "drizzle-kit generate --config=drizzle.sqlite.config.ts", + "drizzle:check:sqlite": "drizzle-kit check --config=drizzle.sqlite.config.ts", + "test": "vitest" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "types": "./dist/types/types.d.ts", + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js" + } + }, + "dependencies": { + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-core": "workspace:*", + "@powersync/service-jsonbig": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "better-sqlite3": "^12.10.0", + "drizzle-orm": "^0.45.2", + "ts-codec": "^1.3.0", + "uuid": "catalog:" + }, + "devDependencies": { + "@powersync/service-core-tests": "workspace:*", + "@types/better-sqlite3": "^7.6.13", + "drizzle-kit": "^0.31.10", + "typescript": "catalog:" + } +} diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/SqliteDrizzleStorageFactory.ts b/modules/module-drizzle-storage/src/drivers/sqlite/SqliteDrizzleStorageFactory.ts new file mode 100644 index 000000000..23408da92 --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/SqliteDrizzleStorageFactory.ts @@ -0,0 +1,17 @@ +import { DrizzleBucketStorageFactory } from '../../storage/DrizzleBucketStorageFactory.js'; +import type { NormalizedDrizzleSqliteStorageConfig } from '../../types/types.js'; +import { createSqliteDrizzleRuntime, type SqliteDrizzleRuntime } from './sqlite-config.js'; +import { createSqliteDrizzleStorageDialect } from './sqlite-dialect.js'; + +export function createSqliteDrizzleStorageFactory(options: { + config: NormalizedDrizzleSqliteStorageConfig; + slotNamePrefix: string; + runtime?: SqliteDrizzleRuntime; +}): DrizzleBucketStorageFactory { + const runtime = options.runtime ?? createSqliteDrizzleRuntime(options.config); + return new DrizzleBucketStorageFactory({ + runtime, + dialect: createSqliteDrizzleStorageDialect(runtime), + slotNamePrefix: options.slotNamePrefix + }); +} diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts b/modules/module-drizzle-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts new file mode 100644 index 000000000..9c83284b9 --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts @@ -0,0 +1,76 @@ +import { locks } from '@powersync/lib-services-framework'; +import * as uuid from 'uuid'; +import type { SqliteDrizzleRuntime } from './sqlite-config.js'; + +const DEFAULT_LOCK_TIMEOUT = 60_000; + +export class SqliteMigrationLockManager extends locks.AbstractLockManager { + constructor(private readonly options: locks.LockManagerParams & { runtime: SqliteDrizzleRuntime }) { + super(options); + } + + private get timeout(): number { + return this.options.timeout ?? DEFAULT_LOCK_TIMEOUT; + } + + async init(): Promise { + this.options.runtime.client.exec(` + CREATE TABLE IF NOT EXISTS powersync_migration_locks ( + name TEXT PRIMARY KEY, + lock_id TEXT, + expires_at INTEGER NOT NULL + ) + `); + this.options.runtime.client + .prepare( + ` + INSERT OR IGNORE INTO powersync_migration_locks (name, lock_id, expires_at) + VALUES (?, NULL, 0) + ` + ) + .run(this.options.name); + } + + protected async acquireHandle(): Promise { + const lockId = uuid.v4(); + const now = Date.now(); + const result = this.options.runtime.client + .prepare( + ` + UPDATE powersync_migration_locks + SET lock_id = ?, expires_at = ? + WHERE name = ? AND (lock_id IS NULL OR expires_at <= ?) + ` + ) + .run(lockId, now + this.timeout, this.options.name, now); + if (result.changes != 1) { + return null; + } + return { + refresh: async () => { + const refreshed = this.options.runtime.client + .prepare( + ` + UPDATE powersync_migration_locks SET expires_at = ? WHERE name = ? AND lock_id = ? + ` + ) + .run(Date.now() + this.timeout, this.options.name, lockId); + if (refreshed.changes != 1) { + throw new Error('Lock not found, could not refresh'); + } + }, + release: async () => { + const released = this.options.runtime.client + .prepare( + ` + UPDATE powersync_migration_locks SET lock_id = NULL, expires_at = 0 WHERE name = ? AND lock_id = ? + ` + ) + .run(this.options.name, lockId); + if (released.changes != 1) { + throw new Error('Lock not found, could not release'); + } + } + }; + } +} diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/column-types.ts b/modules/module-drizzle-storage/src/drivers/sqlite/column-types.ts new file mode 100644 index 000000000..ceb54e8da --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/column-types.ts @@ -0,0 +1,55 @@ +import { customType } from 'drizzle-orm/sqlite-core'; + +/** + * SQLite INTEGER column whose application type is always bigint. + * + * The better-sqlite3 connections are configured with safe integers so the + * driver value reaches this mapper without first losing precision. + */ +export const sqliteBigInt = customType<{ data: bigint; driverData: bigint | number }>({ + dataType() { + return 'bigint'; + }, + toDriver(value) { + return value; + }, + fromDriver(value) { + return BigInt(value); + } +}); + +export const sqliteInteger = customType<{ data: number; driverData: bigint | number }>({ + dataType() { + return 'integer'; + }, + toDriver(value) { + return value; + }, + fromDriver(value) { + return Number(value); + } +}); + +export const sqliteBoolean = customType<{ data: boolean; driverData: bigint | number }>({ + dataType() { + return 'integer'; + }, + toDriver(value) { + return value ? 1 : 0; + }, + fromDriver(value) { + return value !== 0n && value !== 0; + } +}); + +export const sqliteTimestampMs = customType<{ data: Date; driverData: bigint | number }>({ + dataType() { + return 'integer'; + }, + toDriver(value) { + return value.getTime(); + }, + fromDriver(value) { + return new Date(Number(value)); + } +}); diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts b/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts new file mode 100644 index 000000000..cce6e93a1 --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/schema.ts @@ -0,0 +1,148 @@ +import type { storage } from '@powersync/service-core'; +import { blob, index, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { sqliteBigInt, sqliteBoolean, sqliteInteger, sqliteTimestampMs } from './column-types.js'; + +export type CurrentBucket = { + bucket: string; + table: string; + id: string; +}; + +export const bucketData = sqliteTable( + 'bucket_data', + { + id: text('id').primaryKey(), + groupId: sqliteInteger('group_id').notNull(), + bucketName: text('bucket_name').notNull(), + opId: sqliteBigInt('op_id').notNull(), + op: text('op').notNull(), + sourceTable: text('source_table'), + sourceKey: blob('source_key', { mode: 'buffer' }), + tableName: text('table_name'), + rowId: text('row_id'), + checksum: sqliteBigInt('checksum').notNull(), + data: text('data'), + targetOp: sqliteBigInt('target_op') + }, + (table) => [ + index('bucket_data_bucket_op_index').on(table.groupId, table.bucketName, table.opId), + index('bucket_data_source_index').on(table.groupId, table.sourceTable, table.sourceKey) + ] +); + +export const bucketParameters = sqliteTable( + 'bucket_parameters', + { + id: sqliteBigInt('id').primaryKey(), + groupId: sqliteInteger('group_id').notNull(), + sourceTable: text('source_table').notNull(), + sourceKey: blob('source_key', { mode: 'buffer' }).notNull(), + lookup: blob('lookup', { mode: 'buffer' }).notNull(), + bucketParameters: text('bucket_parameters', { mode: 'json' }).$type().notNull() + }, + (table) => [ + index('bucket_parameters_lookup_index').on(table.groupId, table.lookup, table.id), + index('bucket_parameters_source_index').on(table.groupId, table.sourceTable, table.sourceKey) + ] +); + +export const currentData = sqliteTable( + 'current_data', + { + id: text('id').primaryKey(), + groupId: sqliteInteger('group_id').notNull(), + sourceTable: text('source_table').notNull(), + sourceKey: blob('source_key', { mode: 'buffer' }).notNull(), + buckets: text('buckets', { mode: 'json' }).$type().notNull(), + lookups: text('lookups', { mode: 'json' }).$type().notNull(), + data: blob('data', { mode: 'buffer' }).notNull(), + pendingDelete: sqliteBigInt('pending_delete') + }, + (table) => [ + index('current_data_source_index').on(table.groupId, table.sourceTable, table.sourceKey), + index('current_data_pending_delete_index').on(table.groupId, table.pendingDelete) + ] +); + +export const instance = sqliteTable('instance', { + id: text('id').primaryKey() +}); + +export const opIdSequence = sqliteTable('op_id_sequence', { + id: integer('id').primaryKey(), + nextOpId: sqliteBigInt('next_op_id').notNull() +}); + +export const sourceTables = sqliteTable( + 'source_tables', + { + id: text('id').primaryKey(), + groupId: sqliteInteger('group_id').notNull(), + connectionId: sqliteInteger('connection_id').notNull(), + relationId: text('relation_id', { mode: 'json' }).$type(), + schemaName: text('schema_name').notNull(), + tableName: text('table_name').notNull(), + replicaIdColumns: text('replica_id_columns', { mode: 'json' }).$type(), + snapshotDone: sqliteBoolean('snapshot_done').notNull().default(true), + snapshotTotalEstimatedCount: sqliteBigInt('snapshot_total_estimated_count'), + snapshotReplicatedCount: sqliteBigInt('snapshot_replicated_count'), + snapshotLastKey: blob('snapshot_last_key', { mode: 'buffer' }) + }, + (table) => [index('source_table_lookup').on(table.groupId, table.tableName)] +); + +export const syncRules = sqliteTable('sync_rules', { + id: integer('id').primaryKey({ autoIncrement: true }), + state: text('state').$type().notNull(), + snapshotDone: sqliteBoolean('snapshot_done').notNull().default(false), + snapshotLsn: text('snapshot_lsn'), + lastCheckpoint: sqliteBigInt('last_checkpoint'), + lastCheckpointLsn: text('last_checkpoint_lsn'), + noCheckpointBefore: text('no_checkpoint_before'), + slotName: text('slot_name').notNull(), + lastCheckpointTs: sqliteTimestampMs('last_checkpoint_ts'), + lastKeepaliveTs: sqliteTimestampMs('last_keepalive_ts'), + lastFatalError: text('last_fatal_error'), + lastFatalErrorTs: sqliteTimestampMs('last_fatal_error_ts'), + keepaliveOp: sqliteBigInt('keepalive_op'), + storageVersion: sqliteInteger('storage_version'), + content: text('content').notNull(), + syncPlan: text('sync_plan', { mode: 'json' }).$type() +}); + +export const writeCheckpoints = sqliteTable( + 'write_checkpoints', + { + id: text('id').primaryKey(), + syncRulesId: sqliteInteger('sync_rules_id'), + userId: text('user_id').notNull(), + checkpoint: sqliteBigInt('checkpoint').notNull(), + heads: text('heads', { mode: 'json' }).$type>(), + checkpointRequestedAt: sqliteTimestampMs('checkpoint_requested_at'), + createdAt: sqliteTimestampMs('created_at').notNull() + }, + (table) => [ + index('write_checkpoints_user_checkpoint_index').on(table.userId, table.syncRulesId, table.checkpoint), + index('write_checkpoints_requested_at_index').on(table.checkpointRequestedAt) + ] +); + +export const sqliteSchema = { + bucketData, + bucketParameters, + currentData, + instance, + opIdSequence, + sourceTables, + syncRules, + writeCheckpoints +}; + +export type BucketDataRow = typeof bucketData.$inferSelect; +export type BucketParametersRow = typeof bucketParameters.$inferSelect; +export type CurrentDataRow = typeof currentData.$inferSelect; +export type InstanceRow = typeof instance.$inferSelect; +export type OpIdSequenceRow = typeof opIdSequence.$inferSelect; +export type SourceTableRow = typeof sourceTables.$inferSelect; +export type SyncRulesRow = typeof syncRules.$inferSelect; +export type WriteCheckpointRow = typeof writeCheckpoints.$inferSelect; diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-config.ts b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-config.ts new file mode 100644 index 000000000..1fe59fb44 --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-config.ts @@ -0,0 +1,66 @@ +import Database from 'better-sqlite3'; +import type { ExtractTablesWithRelations } from 'drizzle-orm'; +import type { BetterSQLiteTransaction } from 'drizzle-orm/better-sqlite3'; +import { drizzle, type BetterSQLite3Database } from 'drizzle-orm/better-sqlite3'; +import type { NormalizedDrizzleSqliteStorageConfig } from '../../types/types.js'; +import { sqliteSchema } from './schema.js'; + +export type DrizzleStorageDatabase = BetterSQLite3Database & { + $client: Database.Database; +}; +export type DrizzleStorageTransaction = BetterSQLiteTransaction< + typeof sqliteSchema, + ExtractTablesWithRelations +>; + +export interface SqliteDrizzleRuntime { + readonly db: DrizzleStorageDatabase; + readonly client: Database.Database; + readonly readers: readonly DrizzleStorageDatabase[]; + read(): DrizzleStorageDatabase; + transaction(callback: (tx: DrizzleStorageTransaction) => T): T; + close(): void; +} + +export function createSqliteDrizzleRuntime(config: NormalizedDrizzleSqliteStorageConfig): SqliteDrizzleRuntime { + const fileBacked = config.filename != ':memory:'; + const client = openConnection(config.filename, fileBacked); + const db = drizzle(client, { schema: sqliteSchema }); + const readerCount = fileBacked ? Math.max(0, config.max_pool_size - 1) : 0; + const readerClients = Array.from({ length: readerCount }, () => openConnection(config.filename, true, true)); + const readers = readerClients.map((reader) => drizzle(reader, { schema: sqliteSchema })); + let nextReader = 0; + + return { + db, + client, + readers, + read() { + if (readers.length == 0) { + return db; + } + const reader = readers[nextReader % readers.length]; + nextReader++; + return reader; + }, + transaction(callback) { + return db.transaction(callback); + }, + close() { + for (const reader of readerClients) { + reader.close(); + } + client.close(); + } + }; +} + +function openConnection(filename: string, enableWal: boolean, readonly = false): Database.Database { + const client = new Database(filename, { readonly, fileMustExist: readonly }); + client.defaultSafeIntegers(true); + if (enableWal && !readonly) { + client.pragma('journal_mode = WAL'); + } + client.pragma('busy_timeout = 5000'); + return client; +} diff --git a/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts new file mode 100644 index 000000000..17478c67e --- /dev/null +++ b/modules/module-drizzle-storage/src/drivers/sqlite/sqlite-dialect.ts @@ -0,0 +1,80 @@ +import { InProcessDrizzleCheckpointWatcher, type DrizzleStorageDialect } from '../../storage/DrizzleStorageDialect.js'; +import { DRIZZLE_SQLITE_STORAGE_TYPE } from '../../types/types.js'; +import { sqliteSchema } from './schema.js'; +import type { SqliteDrizzleRuntime } from './sqlite-config.js'; + +export function createSqliteDrizzleStorageDialect(runtime: SqliteDrizzleRuntime): DrizzleStorageDialect { + return { + type: DRIZZLE_SQLITE_STORAGE_TYPE, + db: runtime.db, + tables: sqliteSchema, + transaction: (callback) => runtime.transaction(callback), + async *streamBucketDataRows(options) { + if (options.dataBuckets.length == 0) { + return; + } + + await new Promise((resolve) => setImmediate(resolve)); + const db = options.db ?? runtime.read(); + const sortedBuckets = [...options.dataBuckets].sort((a, b) => a.bucket.localeCompare(b.bucket)); + let remaining = options.limit; + const statement = db.$client + .prepare( + ` + SELECT bucket_name, op_id, op, source_table, source_key, + table_name, row_id, checksum, data, target_op + FROM bucket_data + WHERE group_id = ? AND bucket_name = ? AND op_id > ? AND op_id <= ? + ORDER BY op_id ASC + LIMIT ? + ` + ) + .safeIntegers(true); + for (const request of sortedBuckets) { + if (remaining <= 0) { + return; + } + const rows = statement.iterate( + options.groupId, + request.bucket, + request.start, + options.checkpoint, + remaining + ) as Iterable; + for (const row of rows) { + yield mapBucketDataRow(row); + remaining--; + } + } + }, + createCheckpointWatcher: () => new InProcessDrizzleCheckpointWatcher() + }; +} + +interface RawBucketDataRow { + bucket_name: string; + op_id: bigint; + op: string; + source_table: string | null; + source_key: Buffer | null; + table_name: string | null; + row_id: string | null; + checksum: bigint; + data: string | null; + target_op: bigint | null; +} + +function mapBucketDataRow(row: RawBucketDataRow) { + return { + bucketName: row.bucket_name, + opId: row.op_id, + op: row.op, + sourceTable: row.source_table, + sourceKey: row.source_key, + tableName: row.table_name, + rowId: row.row_id, + checksum: row.checksum, + data: row.data, + targetOp: row.target_op + }; +} diff --git a/modules/module-drizzle-storage/src/index.ts b/modules/module-drizzle-storage/src/index.ts new file mode 100644 index 000000000..110fd84d6 --- /dev/null +++ b/modules/module-drizzle-storage/src/index.ts @@ -0,0 +1,10 @@ +export * from './drivers/sqlite/schema.js'; +export * from './drivers/sqlite/sqlite-config.js'; +export * from './drivers/sqlite/sqlite-dialect.js'; +export * from './drivers/sqlite/SqliteDrizzleStorageFactory.js'; +export * from './migrations/DrizzleMigrationAgent.js'; +export * from './module/DrizzleStorageModule.js'; +export * from './storage/storage-index.js'; +export * as storage from './storage/storage-index.js'; +export * from './types/records.js'; +export * from './types/types.js'; diff --git a/modules/module-drizzle-storage/src/migrations/DrizzleMigrationAgent.ts b/modules/module-drizzle-storage/src/migrations/DrizzleMigrationAgent.ts new file mode 100644 index 000000000..5e77222ea --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/DrizzleMigrationAgent.ts @@ -0,0 +1,58 @@ +import * as framework from '@powersync/lib-services-framework'; +import { migrations } from '@powersync/service-core'; +import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import { SqliteMigrationLockManager } from '../drivers/sqlite/SqliteMigrationLockManager.js'; +import { createSqliteDrizzleRuntime, type SqliteDrizzleRuntime } from '../drivers/sqlite/sqlite-config.js'; +import { normalizeDrizzleSqliteStorageConfig, type DrizzleStorageConfigDecoded } from '../types/types.js'; +import { NoOpMigrationStore } from './NoOpMigrationStore.js'; + +export const SQLITE_DRIZZLE_MIGRATIONS_PATH = new URL('./sqlite', import.meta.url).pathname; + +export function runSqliteDrizzleMigrations(runtime: SqliteDrizzleRuntime): void { + migrate(runtime.db, { migrationsFolder: SQLITE_DRIZZLE_MIGRATIONS_PATH }); +} + +export class DrizzleMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent { + store: framework.MigrationStore = new NoOpMigrationStore(); + locks: framework.LockManager; + private readonly runtime: SqliteDrizzleRuntime; + + constructor(config: DrizzleStorageConfigDecoded) { + super(); + this.runtime = createSqliteDrizzleRuntime(normalizeDrizzleSqliteStorageConfig(config)); + this.locks = new SqliteMigrationLockManager({ + name: 'drizzle-migrations', + runtime: this.runtime + }); + } + + getInternalScriptsDir(): string { + return new URL('./scripts', import.meta.url).pathname; + } + + async run(params: framework.RunMigrationParams): Promise { + if (params.direction != framework.Direction.Up) { + throw new Error('Drizzle storage migrations only support the up direction'); + } + await this.locks.init?.(); + const lock = await this.locks.acquire({ + max_wait_ms: params.maxLockWaitMs ?? framework.DEFAULT_MAX_LOCK_WAIT_MS + }); + if (lock == null) { + throw new Error('Could not acquire Drizzle migration lock'); + } + try { + runSqliteDrizzleMigrations(this.runtime); + } finally { + await lock.release(); + } + } + + async loadInternalMigrations(): Promise[]> { + return []; + } + + async [Symbol.asyncDispose](): Promise { + this.runtime.close(); + } +} diff --git a/modules/module-drizzle-storage/src/migrations/NoOpMigrationStore.ts b/modules/module-drizzle-storage/src/migrations/NoOpMigrationStore.ts new file mode 100644 index 000000000..5e7ea01e0 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/NoOpMigrationStore.ts @@ -0,0 +1,9 @@ +import { migrations } from '@powersync/lib-services-framework'; + +export class NoOpMigrationStore implements migrations.MigrationStore { + async load(): Promise { + return undefined; + } + async save(_state: migrations.MigrationState): Promise {} + async clear(): Promise {} +} diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/0000_real_harpoon.sql b/modules/module-drizzle-storage/src/migrations/sqlite/0000_real_harpoon.sql new file mode 100644 index 000000000..a84d542a9 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/0000_real_harpoon.sql @@ -0,0 +1,89 @@ +CREATE TABLE `bucket_data` ( + `id` text PRIMARY KEY NOT NULL, + `group_id` integer NOT NULL, + `bucket_name` text NOT NULL, + `op_id` bigint NOT NULL, + `op` text NOT NULL, + `source_table` text, + `source_key` blob, + `table_name` text, + `row_id` text, + `checksum` bigint NOT NULL, + `data` text, + `target_op` bigint +); +--> statement-breakpoint +CREATE INDEX `bucket_data_bucket_op_index` ON `bucket_data` (`group_id`,`bucket_name`,`op_id`);--> statement-breakpoint +CREATE INDEX `bucket_data_source_index` ON `bucket_data` (`group_id`,`source_table`,`source_key`);--> statement-breakpoint +CREATE TABLE `bucket_parameters` ( + `id` bigint PRIMARY KEY NOT NULL, + `group_id` integer NOT NULL, + `source_table` text NOT NULL, + `source_key` blob NOT NULL, + `lookup` blob NOT NULL, + `bucket_parameters` text NOT NULL +); +--> statement-breakpoint +CREATE INDEX `bucket_parameters_lookup_index` ON `bucket_parameters` (`group_id`,`lookup`,`id`);--> statement-breakpoint +CREATE INDEX `bucket_parameters_source_index` ON `bucket_parameters` (`group_id`,`source_table`,`source_key`);--> statement-breakpoint +CREATE TABLE `current_data` ( + `id` text PRIMARY KEY NOT NULL, + `group_id` integer NOT NULL, + `source_table` text NOT NULL, + `source_key` blob NOT NULL, + `buckets` text NOT NULL, + `lookups` text NOT NULL, + `data` blob NOT NULL, + `pending_delete` bigint +); +--> statement-breakpoint +CREATE INDEX `current_data_source_index` ON `current_data` (`group_id`,`source_table`,`source_key`);--> statement-breakpoint +CREATE INDEX `current_data_pending_delete_index` ON `current_data` (`group_id`,`pending_delete`);--> statement-breakpoint +CREATE TABLE `instance` ( + `id` text PRIMARY KEY NOT NULL +); +--> statement-breakpoint +CREATE TABLE `source_tables` ( + `id` text PRIMARY KEY NOT NULL, + `group_id` integer NOT NULL, + `connection_id` integer NOT NULL, + `relation_id` text, + `schema_name` text NOT NULL, + `table_name` text NOT NULL, + `replica_id_columns` text, + `snapshot_done` integer DEFAULT true NOT NULL, + `snapshot_total_estimated_count` bigint, + `snapshot_replicated_count` bigint, + `snapshot_last_key` blob +); +--> statement-breakpoint +CREATE INDEX `source_table_lookup` ON `source_tables` (`group_id`,`table_name`);--> statement-breakpoint +CREATE TABLE `sync_rules` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `state` text NOT NULL, + `snapshot_done` integer DEFAULT false NOT NULL, + `snapshot_lsn` text, + `last_checkpoint` bigint, + `last_checkpoint_lsn` text, + `no_checkpoint_before` text, + `slot_name` text NOT NULL, + `last_checkpoint_ts` integer, + `last_keepalive_ts` integer, + `last_fatal_error` text, + `last_fatal_error_ts` integer, + `keepalive_op` bigint, + `storage_version` integer, + `content` text NOT NULL, + `sync_plan` text +); +--> statement-breakpoint +CREATE TABLE `write_checkpoints` ( + `id` text PRIMARY KEY NOT NULL, + `sync_rules_id` integer, + `user_id` text NOT NULL, + `checkpoint` bigint NOT NULL, + `heads` text, + `created_at` integer NOT NULL +); +--> statement-breakpoint +CREATE INDEX `write_checkpoints_user_checkpoint_index` ON `write_checkpoints` (`user_id`,`sync_rules_id`,`checkpoint`); \ No newline at end of file diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/0001_true_matthew_murdock.sql b/modules/module-drizzle-storage/src/migrations/sqlite/0001_true_matthew_murdock.sql new file mode 100644 index 000000000..9cafe1d5b --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/0001_true_matthew_murdock.sql @@ -0,0 +1,14 @@ +CREATE TABLE `op_id_sequence` ( + `id` integer PRIMARY KEY NOT NULL, + `next_op_id` bigint NOT NULL +); +--> statement-breakpoint +INSERT INTO `op_id_sequence` (`id`, `next_op_id`) +SELECT 1, MAX(`max_op_id`) + 1 +FROM ( + SELECT COALESCE(MAX(`op_id`), 0) AS `max_op_id` FROM `bucket_data` + UNION ALL + SELECT COALESCE(MAX(`id`), 0) AS `max_op_id` FROM `bucket_parameters` + UNION ALL + SELECT COALESCE(MAX(`pending_delete`), 0) AS `max_op_id` FROM `current_data` +); diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/0002_illegal_maria_hill.sql b/modules/module-drizzle-storage/src/migrations/sqlite/0002_illegal_maria_hill.sql new file mode 100644 index 000000000..2c2b3a146 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/0002_illegal_maria_hill.sql @@ -0,0 +1,2 @@ +ALTER TABLE `write_checkpoints` ADD `checkpoint_requested_at` integer;--> statement-breakpoint +CREATE INDEX `write_checkpoints_requested_at_index` ON `write_checkpoints` (`checkpoint_requested_at`); \ No newline at end of file diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/meta/0000_snapshot.json b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0000_snapshot.json new file mode 100644 index 000000000..3122e1056 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0000_snapshot.json @@ -0,0 +1,555 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "d595e266-c1de-43fe-9e46-2deb2b424122", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "bucket_data": { + "name": "bucket_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_name": { + "name": "bucket_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op_id": { + "name": "op_id", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op": { + "name": "op", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checksum": { + "name": "checksum", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_op": { + "name": "target_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bucket_data_bucket_op_index": { + "name": "bucket_data_bucket_op_index", + "columns": ["group_id", "bucket_name", "op_id"], + "isUnique": false + }, + "bucket_data_source_index": { + "name": "bucket_data_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bucket_parameters": { + "name": "bucket_parameters", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookup": { + "name": "lookup", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_parameters": { + "name": "bucket_parameters", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bucket_parameters_lookup_index": { + "name": "bucket_parameters_lookup_index", + "columns": ["group_id", "lookup", "id"], + "isUnique": false + }, + "bucket_parameters_source_index": { + "name": "bucket_parameters_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "current_data": { + "name": "current_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "buckets": { + "name": "buckets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookups": { + "name": "lookups", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pending_delete": { + "name": "pending_delete", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "current_data_source_index": { + "name": "current_data_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + }, + "current_data_pending_delete_index": { + "name": "current_data_pending_delete_index", + "columns": ["group_id", "pending_delete"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance": { + "name": "instance", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "source_tables": { + "name": "source_tables", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connection_id": { + "name": "connection_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relation_id": { + "name": "relation_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schema_name": { + "name": "schema_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "replica_id_columns": { + "name": "replica_id_columns", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "snapshot_total_estimated_count": { + "name": "snapshot_total_estimated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_replicated_count": { + "name": "snapshot_replicated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_last_key": { + "name": "snapshot_last_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "source_table_lookup": { + "name": "source_table_lookup", + "columns": ["group_id", "table_name"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sync_rules": { + "name": "sync_rules", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "snapshot_lsn": { + "name": "snapshot_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint": { + "name": "last_checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint_lsn": { + "name": "last_checkpoint_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "no_checkpoint_before": { + "name": "no_checkpoint_before", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "slot_name": { + "name": "slot_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_checkpoint_ts": { + "name": "last_checkpoint_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_keepalive_ts": { + "name": "last_keepalive_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error": { + "name": "last_fatal_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error_ts": { + "name": "last_fatal_error_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "keepalive_op": { + "name": "keepalive_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storage_version": { + "name": "storage_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sync_plan": { + "name": "sync_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "write_checkpoints": { + "name": "write_checkpoints", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "sync_rules_id": { + "name": "sync_rules_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checkpoint": { + "name": "checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "heads": { + "name": "heads", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "write_checkpoints_user_checkpoint_index": { + "name": "write_checkpoints_user_checkpoint_index", + "columns": ["user_id", "sync_rules_id", "checkpoint"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/meta/0001_snapshot.json b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0001_snapshot.json new file mode 100644 index 000000000..dd6724aad --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0001_snapshot.json @@ -0,0 +1,579 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "c3a62738-6f52-44ca-b288-5afcb936b63e", + "prevId": "d595e266-c1de-43fe-9e46-2deb2b424122", + "tables": { + "bucket_data": { + "name": "bucket_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_name": { + "name": "bucket_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op_id": { + "name": "op_id", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op": { + "name": "op", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checksum": { + "name": "checksum", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_op": { + "name": "target_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bucket_data_bucket_op_index": { + "name": "bucket_data_bucket_op_index", + "columns": ["group_id", "bucket_name", "op_id"], + "isUnique": false + }, + "bucket_data_source_index": { + "name": "bucket_data_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bucket_parameters": { + "name": "bucket_parameters", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookup": { + "name": "lookup", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_parameters": { + "name": "bucket_parameters", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bucket_parameters_lookup_index": { + "name": "bucket_parameters_lookup_index", + "columns": ["group_id", "lookup", "id"], + "isUnique": false + }, + "bucket_parameters_source_index": { + "name": "bucket_parameters_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "current_data": { + "name": "current_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "buckets": { + "name": "buckets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookups": { + "name": "lookups", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pending_delete": { + "name": "pending_delete", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "current_data_source_index": { + "name": "current_data_source_index", + "columns": ["group_id", "source_table", "source_key"], + "isUnique": false + }, + "current_data_pending_delete_index": { + "name": "current_data_pending_delete_index", + "columns": ["group_id", "pending_delete"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance": { + "name": "instance", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "op_id_sequence": { + "name": "op_id_sequence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "next_op_id": { + "name": "next_op_id", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "source_tables": { + "name": "source_tables", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connection_id": { + "name": "connection_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relation_id": { + "name": "relation_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schema_name": { + "name": "schema_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "replica_id_columns": { + "name": "replica_id_columns", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "snapshot_total_estimated_count": { + "name": "snapshot_total_estimated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_replicated_count": { + "name": "snapshot_replicated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_last_key": { + "name": "snapshot_last_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "source_table_lookup": { + "name": "source_table_lookup", + "columns": ["group_id", "table_name"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sync_rules": { + "name": "sync_rules", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "snapshot_lsn": { + "name": "snapshot_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint": { + "name": "last_checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint_lsn": { + "name": "last_checkpoint_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "no_checkpoint_before": { + "name": "no_checkpoint_before", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "slot_name": { + "name": "slot_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_checkpoint_ts": { + "name": "last_checkpoint_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_keepalive_ts": { + "name": "last_keepalive_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error": { + "name": "last_fatal_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error_ts": { + "name": "last_fatal_error_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "keepalive_op": { + "name": "keepalive_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storage_version": { + "name": "storage_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sync_plan": { + "name": "sync_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "write_checkpoints": { + "name": "write_checkpoints", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "sync_rules_id": { + "name": "sync_rules_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checkpoint": { + "name": "checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "heads": { + "name": "heads", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "write_checkpoints_user_checkpoint_index": { + "name": "write_checkpoints_user_checkpoint_index", + "columns": ["user_id", "sync_rules_id", "checkpoint"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/meta/0002_snapshot.json b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0002_snapshot.json new file mode 100644 index 000000000..4af3bb273 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/meta/0002_snapshot.json @@ -0,0 +1,623 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "3fd174a8-29b8-4e42-9165-cab9c67aea7a", + "prevId": "c3a62738-6f52-44ca-b288-5afcb936b63e", + "tables": { + "bucket_data": { + "name": "bucket_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_name": { + "name": "bucket_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op_id": { + "name": "op_id", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "op": { + "name": "op", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "row_id": { + "name": "row_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checksum": { + "name": "checksum", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_op": { + "name": "target_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bucket_data_bucket_op_index": { + "name": "bucket_data_bucket_op_index", + "columns": [ + "group_id", + "bucket_name", + "op_id" + ], + "isUnique": false + }, + "bucket_data_source_index": { + "name": "bucket_data_source_index", + "columns": [ + "group_id", + "source_table", + "source_key" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bucket_parameters": { + "name": "bucket_parameters", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookup": { + "name": "lookup", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_parameters": { + "name": "bucket_parameters", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bucket_parameters_lookup_index": { + "name": "bucket_parameters_lookup_index", + "columns": [ + "group_id", + "lookup", + "id" + ], + "isUnique": false + }, + "bucket_parameters_source_index": { + "name": "bucket_parameters_source_index", + "columns": [ + "group_id", + "source_table", + "source_key" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "current_data": { + "name": "current_data", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_table": { + "name": "source_table", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "buckets": { + "name": "buckets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lookups": { + "name": "lookups", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pending_delete": { + "name": "pending_delete", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "current_data_source_index": { + "name": "current_data_source_index", + "columns": [ + "group_id", + "source_table", + "source_key" + ], + "isUnique": false + }, + "current_data_pending_delete_index": { + "name": "current_data_pending_delete_index", + "columns": [ + "group_id", + "pending_delete" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance": { + "name": "instance", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "op_id_sequence": { + "name": "op_id_sequence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "next_op_id": { + "name": "next_op_id", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "source_tables": { + "name": "source_tables", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connection_id": { + "name": "connection_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "relation_id": { + "name": "relation_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schema_name": { + "name": "schema_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "table_name": { + "name": "table_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "replica_id_columns": { + "name": "replica_id_columns", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "snapshot_total_estimated_count": { + "name": "snapshot_total_estimated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_replicated_count": { + "name": "snapshot_replicated_count", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_last_key": { + "name": "snapshot_last_key", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "source_table_lookup": { + "name": "source_table_lookup", + "columns": [ + "group_id", + "table_name" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sync_rules": { + "name": "sync_rules", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_done": { + "name": "snapshot_done", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "snapshot_lsn": { + "name": "snapshot_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint": { + "name": "last_checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_checkpoint_lsn": { + "name": "last_checkpoint_lsn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "no_checkpoint_before": { + "name": "no_checkpoint_before", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "slot_name": { + "name": "slot_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_checkpoint_ts": { + "name": "last_checkpoint_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_keepalive_ts": { + "name": "last_keepalive_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error": { + "name": "last_fatal_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fatal_error_ts": { + "name": "last_fatal_error_ts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "keepalive_op": { + "name": "keepalive_op", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storage_version": { + "name": "storage_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sync_plan": { + "name": "sync_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "write_checkpoints": { + "name": "write_checkpoints", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "sync_rules_id": { + "name": "sync_rules_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checkpoint": { + "name": "checkpoint", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "heads": { + "name": "heads", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checkpoint_requested_at": { + "name": "checkpoint_requested_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "write_checkpoints_user_checkpoint_index": { + "name": "write_checkpoints_user_checkpoint_index", + "columns": [ + "user_id", + "sync_rules_id", + "checkpoint" + ], + "isUnique": false + }, + "write_checkpoints_requested_at_index": { + "name": "write_checkpoints_requested_at_index", + "columns": [ + "checkpoint_requested_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json b/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json new file mode 100644 index 000000000..d130ef6b1 --- /dev/null +++ b/modules/module-drizzle-storage/src/migrations/sqlite/meta/_journal.json @@ -0,0 +1,27 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1783955536218, + "tag": "0000_real_harpoon", + "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1784130979161, + "tag": "0001_true_matthew_murdock", + "breakpoints": true + }, + { + "idx": 2, + "version": "6", + "when": 1786603516642, + "tag": "0002_illegal_maria_hill", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/modules/module-drizzle-storage/src/module/DrizzleStorageModule.ts b/modules/module-drizzle-storage/src/module/DrizzleStorageModule.ts new file mode 100644 index 000000000..46a943e11 --- /dev/null +++ b/modules/module-drizzle-storage/src/module/DrizzleStorageModule.ts @@ -0,0 +1,21 @@ +import { modules, system } from '@powersync/service-core'; +import { DrizzleMigrationAgent } from '../migrations/DrizzleMigrationAgent.js'; +import { DrizzleStorageProvider } from '../storage/DrizzleStorageProvider.js'; +import { DrizzleStorageConfig, isDrizzleStorageConfig } from '../types/types.js'; + +export class DrizzleStorageModule extends modules.AbstractModule { + constructor() { + super({ name: 'Drizzle Bucket Storage' }); + } + + async initialize(context: system.ServiceContextContainer): Promise { + context.storageEngine.registerProvider(new DrizzleStorageProvider()); + if (isDrizzleStorageConfig(context.configuration.storage)) { + context.migrations.registerMigrationAgent( + new DrizzleMigrationAgent(DrizzleStorageConfig.decode(context.configuration.storage)) + ); + } + } + + async teardown(): Promise {} +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts b/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts new file mode 100644 index 000000000..4193c634e --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleBucketBatch.ts @@ -0,0 +1,753 @@ +import { BaseObserver, DO_NOT_LOG, Logger, ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { ColumnDescriptor, storage, utils } from '@powersync/service-core'; +import * as sync_rules from '@powersync/service-sync-rules'; +import { and, count, eq, inArray, isNull, lte, ne } from 'drizzle-orm'; +import * as uuid from 'uuid'; +import type { SourceTableRow } from '../drivers/sqlite/schema.js'; +import type { DrizzleStorageTransaction } from '../drivers/sqlite/sqlite-config.js'; +import { DrizzleBucketStorageFactory } from './DrizzleBucketStorageFactory.js'; +import { currentBuckets, currentLookups, DrizzlePersistedBatch } from './DrizzlePersistedBatch.js'; +import { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +export interface DrizzleBucketBatchOptions { + factory: DrizzleBucketStorageFactory; + dialect: DrizzleStorageDialect; + logger: Logger; + syncRules: sync_rules.HydratedSyncConfig; + replicationStreamId: number; + replicationStreamName: string; + lastCheckpointLsn: string | null; + keepaliveOp: bigint | null; + resumeFromLsn: string | null; + storeCurrentData: boolean; + skipExistingRows: boolean; + markRecordUnavailable: storage.BucketStorageMarkRecordUnavailable | undefined; + hooks: storage.StorageHooks | undefined; +} + +const MAX_OPERATION_BATCH_COUNT = 2_000; + +export class DrizzleBucketBatch + extends BaseObserver + implements storage.BucketStorageBatch +{ + [DO_NOT_LOG] = true; + + public last_flushed_op: bigint | null = null; + public resumeFromLsn: string | null; + public readonly skipExistingRows: boolean; + + private lastCheckpointLsnValue: string | null; + private persistedOp: bigint | null; + private readonly pendingOperations: storage.SaveOptions[] = []; + private readonly customWriteCheckpointBatch: storage.CustomWriteCheckpointOptions[] = []; + private needsActivation = true; + + constructor(private readonly options: DrizzleBucketBatchOptions) { + super(); + this.lastCheckpointLsnValue = options.lastCheckpointLsn; + this.resumeFromLsn = options.resumeFromLsn; + this.skipExistingRows = options.skipExistingRows; + this.persistedOp = options.keepaliveOp; + } + + get lastCheckpointLsn(): string | null { + return this.lastCheckpointLsnValue; + } + + async [Symbol.asyncDispose](): Promise { + if (this.customWriteCheckpointBatch.length > 0) { + this.options.logger.warn('Disposing writer with unflushed custom write checkpoints'); + } + super.clearListeners(); + } + + async dispose(): Promise { + await this[Symbol.asyncDispose](); + } + + async resolveTables(options: storage.ResolveTablesOptions): Promise { + const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.options.syncRules; + const { connection_id, source } = options; + const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source; + const normalizedReplicaIdColumns = normalizeReplicaIdColumns(replicaIdColumns); + const relationId = { object_id: objectId }; + const { tables } = this.options.dialect; + + return this.options.dialect.transaction((tx) => { + const existingRows = tx + .select() + .from(tables.sourceTables) + .where( + and( + eq(tables.sourceTables.groupId, this.options.replicationStreamId), + eq(tables.sourceTables.connectionId, connection_id) + ) + ) + .all(); + + let sourceTableRow = + existingRows.find((row) => { + const matchesRelationId = objectId == null || relationObjectId(row.relationId) == objectId; + return ( + row.schemaName == schema && + row.tableName == table && + matchesRelationId && + jsonEquals(row.replicaIdColumns, normalizedReplicaIdColumns) + ); + }) ?? null; + + if (sourceTableRow == null) { + sourceTableRow = tx + .insert(tables.sourceTables) + .values({ + id: options.idGenerator ? String(options.idGenerator()) : uuid.v4(), + groupId: this.options.replicationStreamId, + connectionId: connection_id, + relationId, + schemaName: schema, + tableName: table, + replicaIdColumns: normalizedReplicaIdColumns, + snapshotDone: false, + snapshotTotalEstimatedCount: null, + snapshotReplicatedCount: null, + snapshotLastKey: null + }) + .returning() + .get(); + } + + const sourceTable = sourceTableFromRow(sourceTableRow, connectionTag, syncRules); + sourceTable.storeCurrentData = sendsCompleteRows !== true; + + const dropTables = existingRows + .filter((row) => row.id != sourceTableRow.id) + .filter((row) => { + const matchesTableName = row.schemaName == schema && row.tableName == table; + return objectId == null ? matchesTableName : relationObjectId(row.relationId) == objectId || matchesTableName; + }) + .map((row) => sourceTableFromRow(row, connectionTag, syncRules)); + + return { + tables: [sourceTable], + dropTables + }; + }); + } + + async getSourceTableStatus(table: storage.SourceTable): Promise { + const { db, tables } = this.options.dialect; + const row = db + .select() + .from(tables.sourceTables) + .where( + and( + eq(tables.sourceTables.groupId, this.options.replicationStreamId), + eq(tables.sourceTables.id, String(table.id)) + ) + ) + .get(); + + return row == null ? null : sourceTableFromRow(row, table.ref.connectionTag, this.options.syncRules); + } + + async save(record: storage.SaveOptions): Promise { + const { after, before, sourceTable, tag } = record; + const storeCurrentData = this.options.storeCurrentData && sourceTable.storeCurrentData; + for (const event of this.getTableEvents(sourceTable)) { + this.iterateListeners((cb) => + cb.replicationEvent?.({ + batch: this, + table: sourceTable, + data: { + op: tag, + after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined, + before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined + }, + event + }) + ); + } + + if (!sourceTable.syncData && !sourceTable.syncParameters) { + return null; + } + + this.pendingOperations.push(record); + if (this.pendingOperations.length >= MAX_OPERATION_BATCH_COUNT) { + return this.flush(); + } + return null; + } + + async truncate(sourceTables: storage.SourceTable[]): Promise { + await this.flush(); + + let nextOpId = 0n; + let firstOpId = 0n; + + this.options.dialect.transaction((tx) => { + nextOpId = this.getNextOpId(tx); + firstOpId = nextOpId; + const persistedBatch = this.createPersistedBatch(tx); + for (const table of sourceTables) { + if (!table.syncData && !table.syncParameters) { + continue; + } + + const rows = tx + .select() + .from(this.options.dialect.tables.currentData) + .where( + and( + eq(this.options.dialect.tables.currentData.groupId, this.options.replicationStreamId), + eq(this.options.dialect.tables.currentData.sourceTable, String(table.id)) + ) + ) + .all(); + + for (const row of rows) { + const sourceKey = storage.deserializeReplicaId(Buffer.from(row.sourceKey)); + if (table.syncData) { + nextOpId = persistedBatch.persistBucketData({ + table, + sourceKey, + existingBuckets: currentBuckets(row), + evaluated: [], + nextOpId + }); + } + + if (table.syncParameters) { + nextOpId = persistedBatch.persistParameterData({ + table, + sourceKey, + existingLookups: currentLookups(row), + evaluated: [], + nextOpId + }); + } + } + tx.delete(this.options.dialect.tables.currentData) + .where( + and( + eq(this.options.dialect.tables.currentData.groupId, this.options.replicationStreamId), + eq(this.options.dialect.tables.currentData.sourceTable, String(table.id)) + ) + ) + .run(); + } + this.setNextOpId(tx, nextOpId); + }); + + if (nextOpId == firstOpId) { + return null; + } + + const lastOpId = nextOpId - 1n; + this.persistedOp = lastOpId; + this.last_flushed_op = lastOpId; + this.logFlushedOperations({ + operationCount: sourceTables.length, + firstOpId, + nextOpId, + operationLabel: 'truncate source table' + }); + return { flushed_op: lastOpId }; + } + + async drop(sourceTables: storage.SourceTable[]): Promise { + const result = await this.truncate(sourceTables); + this.options.dialect.transaction((tx) => { + for (const table of sourceTables) { + tx.delete(this.options.dialect.tables.sourceTables) + .where( + and( + eq(this.options.dialect.tables.sourceTables.groupId, this.options.replicationStreamId), + eq(this.options.dialect.tables.sourceTables.id, String(table.id)) + ) + ) + .run(); + } + }); + return result; + } + + async flush(_options?: storage.BatchBucketFlushOptions): Promise { + let result: storage.FlushedResult | null = null; + if (this.pendingOperations.length > 0) { + await this.options.hooks?.beforeBatchFlush?.(this); + const operations = this.pendingOperations.splice(0); + let nextOpId = 0n; + let firstOpId: bigint | null = null; + for (const batch of chunked(operations, MAX_OPERATION_BATCH_COUNT)) { + this.options.dialect.transaction((tx) => { + nextOpId = this.getNextOpId(tx); + firstOpId ??= nextOpId; + const persistedBatch = this.createPersistedBatch(tx); + nextOpId = persistedBatch.persistOperations(batch, nextOpId); + this.setNextOpId(tx, nextOpId); + }); + } + const lastOpId = nextOpId - 1n; + this.persistedOp = lastOpId; + this.last_flushed_op = lastOpId; + result = { flushed_op: lastOpId }; + this.logFlushedOperations({ + operationCount: operations.length, + firstOpId: firstOpId!, + nextOpId + }); + await this.options.hooks?.afterBatchFlush?.(this); + } + + await this.flushCustomWriteCheckpoints(); + return result; + } + + async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { + await this.flush(); + const createEmptyCheckpoints = options?.createEmptyCheckpoints ?? true; + const now = new Date(); + const { tables } = this.options.dialect; + + const result = this.options.dialect.transaction((tx) => { + const syncRulesRow = tx + .select() + .from(tables.syncRules) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .get(); + if (syncRulesRow == null) { + throw new Error(`Missing replication stream ${this.options.replicationStreamId}`); + } + + const canCheckpoint = + syncRulesRow.snapshotDone === true && + (syncRulesRow.lastCheckpointLsn == null || syncRulesRow.lastCheckpointLsn <= lsn) && + (syncRulesRow.noCheckpointBefore == null || syncRulesRow.noCheckpointBefore <= lsn); + + let checkpointCreated = false; + + if (canCheckpoint) { + const newLastCheckpoint = maxBigint( + syncRulesRow.lastCheckpoint, + this.persistedOp, + syncRulesRow.keepaliveOp, + 0n + ); + const changed = syncRulesRow.lastCheckpoint !== newLastCheckpoint || syncRulesRow.keepaliveOp != null; + + if (changed || createEmptyCheckpoints) { + tx.update(tables.syncRules) + .set({ + lastCheckpointLsn: lsn, + lastCheckpointTs: now, + lastKeepaliveTs: now, + lastFatalError: null, + keepaliveOp: null, + lastCheckpoint: newLastCheckpoint, + snapshotLsn: null + }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + checkpointCreated = true; + tx.delete(tables.currentData) + .where( + and( + eq(tables.currentData.groupId, this.options.replicationStreamId), + lte(tables.currentData.pendingDelete, newLastCheckpoint) + ) + ) + .run(); + } else { + tx.update(tables.syncRules) + .set({ lastKeepaliveTs: now }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + } + } else { + tx.update(tables.syncRules) + .set({ + keepaliveOp: maxBigint(syncRulesRow.keepaliveOp, this.persistedOp, 0n), + lastKeepaliveTs: now + }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + } + + return { + checkpointBlocked: !canCheckpoint, + checkpointCreated + }; + }); + + if (!result.checkpointBlocked) { + await this.autoActivate(lsn); + } + + this.persistedOp = null; + this.lastCheckpointLsnValue = lsn; + this.options.factory.checkpointWatcher.notify(); + return result; + } + + keepalive(lsn: string): Promise { + return this.commit(lsn, { createEmptyCheckpoints: true }); + } + + async setResumeLsn(lsn: string): Promise { + const { db, tables } = this.options.dialect; + const result = db + .update(tables.syncRules) + .set({ snapshotLsn: lsn }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + if (result.changes == 0) { + throw new Error(`Missing replication stream ${this.options.replicationStreamId}`); + } + this.resumeFromLsn = lsn; + } + + async markAllSnapshotDone(noCheckpointBeforeLsn: string): Promise { + await this.markSnapshotDoneInternal(noCheckpointBeforeLsn); + } + + async markSnapshotDone(noCheckpointBeforeLsn: string, options?: { throwOnConflict?: boolean }): Promise { + const { db, tables } = this.options.dialect; + const remaining = + db + .select({ value: count() }) + .from(tables.sourceTables) + .where( + and( + eq(tables.sourceTables.groupId, this.options.replicationStreamId), + eq(tables.sourceTables.snapshotDone, false) + ) + ) + .get()?.value ?? 0; + + if (remaining > 0) { + if (options?.throwOnConflict ?? true) { + throw new ReplicationAssertionError( + `Cannot mark snapshot done while ${remaining} source table${remaining == 1 ? '' : 's'} still require snapshotting` + ); + } + return; + } + + await this.markSnapshotDoneInternal(noCheckpointBeforeLsn); + } + + async markTableSnapshotRequired(_table: storage.SourceTable): Promise { + const { db, tables } = this.options.dialect; + const result = db + .update(tables.syncRules) + .set({ snapshotDone: false }) + .where(eq(tables.syncRules.id, this.options.replicationStreamId)) + .run(); + if (result.changes == 0) { + throw new Error(`Missing replication stream ${this.options.replicationStreamId}`); + } + } + + async markTableSnapshotDone( + tables: storage.SourceTable[], + noCheckpointBeforeLsn?: string + ): Promise { + const ids = tables.map((table) => String(table.id)); + this.options.dialect.transaction((tx) => { + if (ids.length > 0) { + tx.update(this.options.dialect.tables.sourceTables) + .set({ + snapshotDone: true, + snapshotTotalEstimatedCount: null, + snapshotReplicatedCount: null, + snapshotLastKey: null + }) + .where(inArray(this.options.dialect.tables.sourceTables.id, ids)) + .run(); + } + + if (noCheckpointBeforeLsn != null) { + this.assignNoCheckpointBefore(tx, noCheckpointBeforeLsn); + } + }); + + return tables.map((table) => { + const copy = table.clone(); + copy.snapshotComplete = true; + copy.snapshotStatus = undefined; + return copy; + }); + } + + async updateTableProgress( + table: storage.SourceTable, + progress: Partial + ): Promise { + const copy = table.clone(); + const snapshotStatus = { + totalEstimatedCount: progress.totalEstimatedCount ?? copy.snapshotStatus?.totalEstimatedCount ?? 0, + replicatedCount: progress.replicatedCount ?? copy.snapshotStatus?.replicatedCount ?? 0, + lastKey: progress.lastKey ?? copy.snapshotStatus?.lastKey ?? null + }; + copy.snapshotStatus = snapshotStatus; + + const { db, tables } = this.options.dialect; + const result = db + .update(tables.sourceTables) + .set({ + snapshotTotalEstimatedCount: BigInt(snapshotStatus.totalEstimatedCount), + snapshotReplicatedCount: BigInt(snapshotStatus.replicatedCount), + snapshotLastKey: snapshotStatus.lastKey == null ? null : Buffer.from(snapshotStatus.lastKey) + }) + .where(eq(tables.sourceTables.id, String(table.id))) + .run(); + if (result.changes == 0) { + throw new Error(`Missing source table ${table.id}`); + } + + return copy; + } + + addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void { + this.customWriteCheckpointBatch.push({ + ...checkpoint, + sync_rules_id: this.options.replicationStreamId + }); + } + + private async flushCustomWriteCheckpoints(): Promise { + if (this.customWriteCheckpointBatch.length == 0) { + return; + } + + const batch = this.customWriteCheckpointBatch.splice(0); + this.options.dialect.transaction((tx) => { + for (const checkpoint of batch) { + const table = this.options.dialect.tables.writeCheckpoints; + const existing = tx + .select() + .from(table) + .where( + and( + eq(table.userId, checkpoint.user_id), + checkpoint.sync_rules_id == null + ? isNull(table.syncRulesId) + : eq(table.syncRulesId, checkpoint.sync_rules_id) + ) + ) + .get(); + + if (existing == null) { + tx.insert(table) + .values({ + id: uuid.v4(), + syncRulesId: checkpoint.sync_rules_id, + userId: checkpoint.user_id, + checkpoint: checkpoint.checkpoint, + heads: null, + checkpointRequestedAt: checkpoint.checkpoint_requested_at ?? null, + createdAt: new Date() + }) + .run(); + } else { + tx.update(table) + .set({ + checkpoint: checkpoint.checkpoint, + checkpointRequestedAt: checkpoint.checkpoint_requested_at ?? null, + createdAt: new Date() + }) + .where(eq(table.id, existing.id)) + .run(); + } + } + }); + this.options.factory.checkpointWatcher.notify(); + } + + private createPersistedBatch(tx: DrizzleStorageTransaction): DrizzlePersistedBatch { + return new DrizzlePersistedBatch({ + tx, + dialect: this.options.dialect, + logger: this.options.logger, + syncRules: this.options.syncRules, + replicationStreamId: this.options.replicationStreamId, + storeCurrentData: this.options.storeCurrentData, + skipExistingRows: this.options.skipExistingRows, + markRecordUnavailable: this.options.markRecordUnavailable + }); + } + + private getNextOpId(tx: DrizzleStorageTransaction): bigint { + const row = tx + .select() + .from(this.options.dialect.tables.opIdSequence) + .where(eq(this.options.dialect.tables.opIdSequence.id, 1)) + .get(); + if (row == null) { + throw new Error('Missing op ID sequence state'); + } + return row.nextOpId; + } + + private setNextOpId(tx: DrizzleStorageTransaction, nextOpId: bigint): void { + tx.update(this.options.dialect.tables.opIdSequence) + .set({ nextOpId }) + .where(eq(this.options.dialect.tables.opIdSequence.id, 1)) + .run(); + } + + private logFlushedOperations(options: { + operationCount: number; + firstOpId: bigint; + nextOpId: bigint; + operationLabel?: string; + }): void { + const storageOperationCount = options.nextOpId - options.firstOpId; + const operationLabel = options.operationLabel ?? 'source operation'; + const pluralizedOperationLabel = options.operationCount == 1 ? operationLabel : `${operationLabel}s`; + + if (storageOperationCount == 0n) { + this.options.logger.info( + `[${this.options.replicationStreamName}] Flushed ${options.operationCount} ${pluralizedOperationLabel} to Drizzle storage DB with no new storage ops` + ); + return; + } + + this.options.logger.info( + `[${this.options.replicationStreamName}] Flushed ${options.operationCount} ${pluralizedOperationLabel} to Drizzle storage DB as ${storageOperationCount.toString()} storage ops (${options.firstOpId.toString()}-${(options.nextOpId - 1n).toString()})` + ); + } + + private async markSnapshotDoneInternal(noCheckpointBeforeLsn: string): Promise { + this.options.dialect.transaction((tx) => { + this.assignNoCheckpointBefore(tx, noCheckpointBeforeLsn); + }); + this.options.factory.checkpointWatcher.notify(); + } + + private assignNoCheckpointBefore(tx: DrizzleStorageTransaction, noCheckpointBeforeLsn: string): void { + const table = this.options.dialect.tables.syncRules; + const row = tx.select().from(table).where(eq(table.id, this.options.replicationStreamId)).get(); + if (row == null) { + throw new Error(`Missing replication stream ${this.options.replicationStreamId}`); + } + tx.update(table) + .set({ + snapshotDone: true, + lastKeepaliveTs: new Date(), + noCheckpointBefore: + row.noCheckpointBefore == null || row.noCheckpointBefore < noCheckpointBeforeLsn + ? noCheckpointBeforeLsn + : row.noCheckpointBefore + }) + .where(eq(table.id, this.options.replicationStreamId)) + .run(); + } + + private async autoActivate(lsn: string): Promise { + if (!this.needsActivation) { + return; + } + + let didActivate = false; + this.options.dialect.transaction((tx) => { + const table = this.options.dialect.tables.syncRules; + const syncRulesRow = tx.select().from(table).where(eq(table.id, this.options.replicationStreamId)).get(); + + if (syncRulesRow?.state == storage.SyncRuleState.PROCESSING && syncRulesRow.snapshotDone) { + tx.update(table) + .set({ state: storage.SyncRuleState.ACTIVE }) + .where(eq(table.id, this.options.replicationStreamId)) + .run(); + tx.update(table) + .set({ state: storage.SyncRuleState.STOP }) + .where( + and( + ne(table.id, this.options.replicationStreamId), + inArray(table.state, [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED]) + ) + ) + .run(); + didActivate = true; + this.needsActivation = false; + } else if (syncRulesRow?.state != storage.SyncRuleState.PROCESSING) { + this.needsActivation = false; + } + }); + + if (didActivate) { + this.options.logger.info(`Activated new replication stream at ${lsn}`); + } + } + + private getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] { + return this.options.syncRules.eventDescriptors.filter((event) => + [...event.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref)) + ); + } +} + +function sourceTableFromRow( + row: SourceTableRow, + connectionTag: string, + syncRules: sync_rules.HydratedSyncConfig +): storage.SourceTable { + const ref = { connectionTag, schema: row.schemaName, name: row.tableName }; + const sourceTable = new storage.SourceTable({ + id: row.id, + ref, + objectId: relationObjectId(row.relationId), + replicaIdColumns: replicaIdColumns(row.replicaIdColumns), + snapshotComplete: row.snapshotDone ?? true, + ...syncRules.getMatchingSources(ref) + }); + + if (!sourceTable.snapshotComplete) { + sourceTable.snapshotStatus = { + totalEstimatedCount: Number(row.snapshotTotalEstimatedCount ?? -1n), + replicatedCount: Number(row.snapshotReplicatedCount ?? 0n), + lastKey: row.snapshotLastKey ?? null + }; + } + + sourceTable.syncEvent = syncRules.tableTriggersEvent(ref); + sourceTable.syncData = sourceTable.bucketDataSources.length > 0; + sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0; + return sourceTable; +} + +function normalizeReplicaIdColumns(replicaIdColumns: ColumnDescriptor[]): ColumnDescriptor[] { + return replicaIdColumns.map((column) => ({ + name: column.name, + type: column.type, + typeId: typeof column.typeId === 'undefined' ? column.typeId : Number(column.typeId) + })); +} + +function replicaIdColumns(value: unknown): ColumnDescriptor[] { + return Array.isArray(value) ? (value as ColumnDescriptor[]) : []; +} + +function relationObjectId(value: unknown): string | number | undefined { + if (value == null || typeof value != 'object' || Array.isArray(value)) { + return undefined; + } + const objectId = (value as Record).object_id; + return typeof objectId == 'string' || typeof objectId == 'number' ? objectId : undefined; +} + +function jsonEquals(left: unknown, right: unknown): boolean { + return JSON.stringify(left ?? null) == JSON.stringify(right ?? null); +} + +function maxBigint(...values: (bigint | null | undefined)[]): bigint { + return values.reduce((max, value) => (value != null && value > max ? value : max), 0n); +} + +function* chunked(items: T[], size: number): Generator { + for (let index = 0; index < items.length; index += size) { + yield items.slice(index, index + size); + } +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleBucketStorageFactory.ts b/modules/module-drizzle-storage/src/storage/DrizzleBucketStorageFactory.ts new file mode 100644 index 000000000..2fd293595 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleBucketStorageFactory.ts @@ -0,0 +1,239 @@ +import { DO_NOT_LOG } from '@powersync/lib-services-framework'; +import { GetIntanceOptions, storage } from '@powersync/service-core'; +import crypto from 'crypto'; +import { desc, eq, inArray } from 'drizzle-orm'; +import * as uuid from 'uuid'; +import type { BucketDataRow, BucketParametersRow, CurrentDataRow, SyncRulesRow } from '../drivers/sqlite/schema.js'; +import type { SqliteDrizzleRuntime } from '../drivers/sqlite/sqlite-config.js'; +import { DrizzlePersistedReplicationStream } from './DrizzlePersistedReplicationStream.js'; +import type { DrizzleCheckpointWatcher, DrizzleStorageDialect } from './DrizzleStorageDialect.js'; +import { DrizzleSyncRulesStorage } from './DrizzleSyncRulesStorage.js'; + +export interface DrizzleBucketStorageFactoryOptions { + runtime: SqliteDrizzleRuntime; + dialect: DrizzleStorageDialect; + slotNamePrefix: string; +} + +export class DrizzleBucketStorageFactory extends storage.BucketStorageFactory { + [DO_NOT_LOG] = true; + + readonly runtime: SqliteDrizzleRuntime; + readonly dialect: DrizzleStorageDialect; + readonly slotNamePrefix: string; + readonly checkpointWatcher: DrizzleCheckpointWatcher; + + constructor(options: DrizzleBucketStorageFactoryOptions) { + super(); + this.runtime = options.runtime; + this.dialect = options.dialect; + this.slotNamePrefix = options.slotNamePrefix; + this.checkpointWatcher = options.dialect.createCheckpointWatcher(); + } + + async [Symbol.asyncDispose](): Promise { + this.runtime.close(); + } + + getInstance( + replicationStream: storage.PersistedReplicationStream, + options?: GetIntanceOptions + ): storage.SyncRulesBucketStorage { + const syncRuleStorage = new DrizzleSyncRulesStorage({ + factory: this, + dialect: this.dialect, + replicationStream + }); + + if (!options?.skipLifecycleHooks) { + this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage)); + } + + syncRuleStorage.registerListener({ + batchStarted: (batch) => { + batch.registerListener({ + replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) + }); + } + }); + + return syncRuleStorage; + } + + async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { + const storageVersion = + options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; + if (storage.STORAGE_VERSION_CONFIG[storageVersion] == null) { + throw new Error(`Unsupported storage version ${storageVersion}`); + } + + const tables = this.dialect.tables; + const row = this.dialect.transaction((tx) => { + tx.update(tables.syncRules) + .set({ state: storage.SyncRuleState.STOP }) + .where(eq(tables.syncRules.state, storage.SyncRuleState.PROCESSING)) + .run(); + + return tx + .insert(tables.syncRules) + .values({ + state: storage.SyncRuleState.PROCESSING, + snapshotDone: false, + snapshotLsn: null, + lastCheckpoint: null, + lastCheckpointLsn: null, + noCheckpointBefore: null, + slotName: this.generateReplicationStreamName(), + lastCheckpointTs: null, + lastKeepaliveTs: null, + lastFatalError: null, + lastFatalErrorTs: null, + keepaliveOp: null, + storageVersion, + content: options.config.yaml, + syncPlan: options.config.plan + }) + .returning() + .get(); + }); + + return new DrizzlePersistedReplicationStream(this.dialect, row); + } + + async restartReplication(replicationStreamId: number): Promise { + const active = await this.getActiveSyncConfig(); + const deploying = await this.getDeployingSyncConfig(); + const stream = + deploying?.replicationStream.replicationStreamId == replicationStreamId + ? deploying + : active?.replicationStream.replicationStreamId == replicationStreamId + ? active + : null; + if (stream != null) { + await this.updateSyncRules(stream.content.asUpdateOptions()); + } + } + + async getActiveSyncConfig(): Promise { + return this.getSyncConfigForStates([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED]); + } + + async getDeployingSyncConfig(): Promise { + return this.getSyncConfigForStates([storage.SyncRuleState.PROCESSING]); + } + + async getReplicatingReplicationStreams(): Promise { + return this.findSyncRulesRows([storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]).map( + (row) => new DrizzlePersistedReplicationStream(this.dialect, row) + ); + } + + async getStoppedReplicationStreams(): Promise { + return this.findSyncRulesRows([storage.SyncRuleState.STOP]).map( + (row) => new DrizzlePersistedReplicationStream(this.dialect, row) + ); + } + + async getStorageMetrics(): Promise { + const { db, tables } = this.dialect; + const operations = db.select().from(tables.bucketData).all(); + const parameters = db.select().from(tables.bucketParameters).all(); + const currentData = db.select().from(tables.currentData).all(); + return { + operations_size_bytes: operations.reduce((total, row) => total + estimateBucketDataSize(row), 0), + parameters_size_bytes: parameters.reduce((total, row) => total + estimateBucketParametersSize(row), 0), + replication_size_bytes: currentData.reduce((total, row) => total + estimateCurrentDataSize(row), 0) + }; + } + + async getPowerSyncInstanceId(): Promise { + const { db, tables } = this.dialect; + const existing = db.select().from(tables.instance).limit(1).get(); + if (existing != null) { + return existing.id; + } + const id = uuid.v4(); + db.insert(tables.instance).values({ id }).onConflictDoNothing().run(); + return db.select().from(tables.instance).limit(1).get()!.id; + } + + async getSystemIdentifier(): Promise { + return { + id: `${this.dialect.type}:${await this.getPowerSyncInstanceId()}`, + type: this.dialect.type + }; + } + + private async getSyncConfigForStates(states: storage.SyncRuleState[]): Promise { + const [row] = this.findSyncRulesRows(states); + if (row == null) { + return null; + } + const replicationStream = new DrizzlePersistedReplicationStream(this.dialect, row); + return { + content: replicationStream.syncConfigContent[0], + replicationStream, + storage: this.getInstance(replicationStream, { skipLifecycleHooks: true }) + }; + } + + private findSyncRulesRows(states: storage.SyncRuleState[]): SyncRulesRow[] { + const { db, tables } = this.dialect; + if (states.length == 0) { + return []; + } + return db + .select() + .from(tables.syncRules) + .where(inArray(tables.syncRules.state, states)) + .orderBy(desc(tables.syncRules.id)) + .all(); + } + + private generateReplicationStreamName(): string { + return `${this.slotNamePrefix}${Date.now()}_${crypto.randomBytes(2).toString('hex')}`; + } +} + +function estimateString(value: string | null | undefined): number { + return value == null ? 0 : value.length; +} +function estimateBuffer(value: Buffer | Uint8Array | null | undefined): number { + return value == null ? 0 : value.byteLength; +} +function estimateJson(value: unknown): number { + return typeof value == 'string' ? value.length : JSON.stringify(value ?? null).length; +} +function estimateBucketDataSize(row: BucketDataRow): number { + return ( + 80 + + estimateString(row.id) + + estimateString(row.bucketName) + + estimateString(row.op) + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateString(row.tableName) + + estimateString(row.rowId) + + estimateString(row.data) + ); +} +function estimateBucketParametersSize(row: BucketParametersRow): number { + return ( + 80 + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateBuffer(row.lookup) + + estimateJson(row.bucketParameters) + ); +} +function estimateCurrentDataSize(row: CurrentDataRow): number { + return ( + 80 + + estimateString(row.id) + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateJson(row.buckets) + + estimateJson(row.lookups) + + estimateBuffer(row.data) + ); +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleCompactor.ts b/modules/module-drizzle-storage/src/storage/DrizzleCompactor.ts new file mode 100644 index 000000000..99ab4f570 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleCompactor.ts @@ -0,0 +1,510 @@ +import { Logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import { lt } from 'drizzle-orm'; +import * as uuid from 'uuid'; +import type { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +interface CurrentBucketState { + bucket: string; + seen: Map; + trackingSize: number; + lastNotPut: bigint | null; + opsSincePut: number; +} + +interface ParameterCompactionRow { + id: bigint; + sourceTable: string; + sourceKey: Buffer; + lookup: Buffer; + bucketParameters: unknown; +} + +interface RawParameterCompactionRow { + id: bigint | number | string; + source_table?: string; + sourceTable?: string; + source_key?: Buffer | Uint8Array; + sourceKey?: Buffer | Uint8Array; + lookup: Buffer | Uint8Array; + bucket_parameters?: unknown; + bucketParameters?: unknown; +} + +interface RawBucketDataRow { + id: string; + bucket_name: string; + op_id: bigint | number | string; + op: string; + source_table: string | null; + source_key: Buffer | Uint8Array | null; + table_name: string | null; + row_id: string | null; + checksum: bigint | number | string; + target_op: bigint | number | string | null; +} + +interface CompactionBucketDataRow { + id: string; + bucketName: string; + opId: bigint; + op: string; + sourceTable: string | null; + sourceKey: Buffer | null; + tableName: string | null; + rowId: string | null; + checksum: bigint; + targetOp: bigint | null; +} + +export interface DrizzleCompactOptions extends storage.CompactOptions { + logger: Logger; +} + +const BIGINT_MAX = 9223372036854775807n; +const DEFAULT_CLEAR_BATCH_LIMIT = 5000; +const DEFAULT_MOVE_BATCH_LIMIT = 2000; +const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; +const DEFAULT_MEMORY_LIMIT_MB = 64; +const PARAMETER_DELETE_BATCH_LIMIT = 1000; +const PARAMETER_SCAN_BATCH_LIMIT = 10_000; + +export class DrizzleCompactor { + private readonly idLimitBytes: number; + private readonly moveBatchLimit: number; + private readonly moveBatchQueryLimit: number; + private readonly clearBatchLimit: number; + private readonly maxOpId: bigint; + private readonly buckets: string[] | undefined; + private readonly deleteCheckpointRequestsBefore: Date | undefined; + private readonly logger: Logger; + + private pendingMoves: { id: string; targetOp: bigint }[] = []; + + constructor( + private readonly dialect: DrizzleStorageDialect, + private readonly groupId: number, + options: DrizzleCompactOptions + ) { + this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; + this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; + this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; + this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; + this.maxOpId = options.maxOpId ?? 0n; + this.buckets = options.compactBuckets; + this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore; + this.logger = options.logger; + } + + async compact(): Promise { + this.deleteOldCheckpointRequests(); + + if (this.maxOpId <= 0n) { + return; + } + + if (this.buckets != null) { + for (const bucket of this.buckets) { + await this.compactSingleBucket(bucket); + } + } else { + await this.compactAllBuckets(); + } + } + + private deleteOldCheckpointRequests(): void { + if (this.deleteCheckpointRequestsBefore == null) { + return; + } + + const table = this.dialect.tables.writeCheckpoints; + this.dialect.db.delete(table).where(lt(table.checkpointRequestedAt, this.deleteCheckpointRequestsBefore)).run(); + } + + async compactParameterData(options: storage.CompactOptions): Promise { + if (this.maxOpId <= 0n) { + return; + } + + const lastByKey = new Map(); + const removeIds = new Set(); + const keysWithDuplicateValues = new Set(); + const maxCacheSize = options.compactParameterCacheLimit ?? 10_000; + const flushDeletes = async (force: boolean) => { + if (removeIds.size < PARAMETER_DELETE_BATCH_LIMIT && !(force && removeIds.size > 0)) { + return; + } + + const removedCount = removeIds.size; + for (const id of removeIds) { + await this.executeRun( + ` + DELETE FROM bucket_parameters + WHERE group_id = ? AND id = ? + `, + [this.groupId, id.toString()] + ); + } + removeIds.clear(); + this.logger.info(`Removed ${removedCount} compacted parameter entries`); + }; + + for await (const row of this.streamParameterRows()) { + const key = parameterKey(row); + const previous = lastByKey.get(key); + if (previous != null && sameBucketParameters(previous.bucketParameters, row.bucketParameters)) { + removeIds.add(row.id); + keysWithDuplicateValues.add(key); + } + + if (isEmptyBucketParameters(row.bucketParameters) && row.id < this.maxOpId && !keysWithDuplicateValues.has(key)) { + await flushDeletes(true); + const candidate = lastByKey.get(key); + await this.executeRun( + ` + DELETE FROM bucket_parameters + WHERE group_id = ? + AND lookup = ? + AND source_table = ? + AND source_key = ? + AND id <= ? + `, + [this.groupId, row.lookup, row.sourceTable, row.sourceKey, row.id.toString()] + ); + if (candidate != null && candidate.id <= row.id) { + removeIds.add(candidate.id); + } + removeIds.add(row.id); + lastByKey.delete(key); + } else { + lastByKey.set(key, row); + } + + if (lastByKey.size > maxCacheSize) { + const oldest = lastByKey.keys().next().value; + if (oldest != null) { + lastByKey.delete(oldest); + } + } + + await flushDeletes(false); + } + await flushDeletes(true); + lastByKey.clear(); + } + + private async *streamParameterRows(): AsyncIterable { + let lastRow: ParameterCompactionRow | null = null; + + while (true) { + const params: unknown[] = [this.groupId, this.maxOpId]; + let cursorFilter = ''; + if (lastRow != null) { + cursorFilter = ` + AND ( + lookup > ? + OR (lookup = ? AND source_table > ?) + OR (lookup = ? AND source_table = ? AND source_key > ?) + OR (lookup = ? AND source_table = ? AND source_key = ? AND id > ?) + ) + `; + params.push( + lastRow.lookup, + lastRow.lookup, + lastRow.sourceTable, + lastRow.lookup, + lastRow.sourceTable, + lastRow.sourceKey, + lastRow.lookup, + lastRow.sourceTable, + lastRow.sourceKey, + lastRow.id + ); + } + params.push(PARAMETER_SCAN_BATCH_LIMIT); + + const rows = await this.executeAll( + ` + SELECT id, source_table, source_key, lookup, bucket_parameters + FROM bucket_parameters + WHERE group_id = ? + AND id <= ? + ${cursorFilter} + ORDER BY lookup ASC, source_table ASC, source_key ASC, id ASC + LIMIT ? + `, + params + ); + + if (rows.length == 0) { + return; + } + + for (const rawRow of rows) { + lastRow = rawParameterRow(rawRow); + yield lastRow; + } + } + } + + private async compactAllBuckets(): Promise { + const discoveryBatchSize = 200; + let lastBucket = ''; + + while (true) { + const rows = await this.executeAll<{ bucket_name: string }>( + ` + SELECT DISTINCT bucket_name + FROM bucket_data + WHERE group_id = ? AND bucket_name > ? + ORDER BY bucket_name ASC + LIMIT ? + `, + [this.groupId, lastBucket, discoveryBatchSize] + ); + if (rows.length == 0) { + break; + } + + for (const row of rows) { + await this.compactSingleBucket(row.bucket_name); + } + lastBucket = rows[rows.length - 1].bucket_name; + } + } + + private async compactSingleBucket(bucket: string): Promise { + const currentState: CurrentBucketState = { + bucket, + seen: new Map(), + trackingSize: 0, + lastNotPut: null, + opsSincePut: 0 + }; + let upperOpIdLimit = BIGINT_MAX; + + while (true) { + const batch = ( + await this.executeAll( + ` + SELECT id, bucket_name, op_id, op, source_table, source_key, table_name, row_id, checksum, target_op + FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id < ? + AND op_id <= ? + ORDER BY op_id DESC + LIMIT ? + `, + [this.groupId, bucket, upperOpIdLimit, this.maxOpId, this.moveBatchQueryLimit] + ) + ).map(rawBucketDataRow); + + if (batch.length == 0) { + break; + } + + upperOpIdLimit = batch[batch.length - 1].opId; + + for (const row of batch) { + let isPersistentPut = row.op == 'PUT'; + + if (row.op == 'REMOVE' || row.op == 'PUT') { + const key = compactBucketRowKey(row); + const targetOp = currentState.seen.get(utils.flatstr(key)); + if (targetOp != null) { + isPersistentPut = false; + this.pendingMoves.push({ id: row.id, targetOp }); + } else if (currentState.trackingSize < this.idLimitBytes) { + currentState.seen.set(utils.flatstr(key), row.opId); + currentState.trackingSize += key.length + 140; + } + } + + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (row.op != 'CLEAR') { + currentState.lastNotPut ??= row.opId; + currentState.opsSincePut += 1; + } + + if (this.pendingMoves.length >= this.moveBatchLimit) { + await this.flushMoves(); + } + } + } + + await this.flushMoves(); + currentState.seen.clear(); + if (currentState.lastNotPut != null && currentState.opsSincePut > 1) { + this.logger.info( + `Inserting CLEAR at ${this.groupId}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + await this.clearBucket(currentState.bucket, currentState.lastNotPut); + } + } + + private async flushMoves(): Promise { + if (this.pendingMoves.length == 0) { + return; + } + + const batch = this.pendingMoves.splice(0); + this.logger.info(`Compacting ${batch.length} ops`); + for (const { id, targetOp } of batch) { + await this.executeRun( + ` + UPDATE bucket_data + SET op = 'MOVE', + target_op = ?, + table_name = NULL, + row_id = NULL, + data = NULL, + source_table = NULL, + source_key = NULL + WHERE id = ? + `, + [targetOp, id] + ); + } + } + + private async clearBucket(bucket: string, op: bigint): Promise { + let done = false; + while (!done) { + const operationRows = ( + await this.executeAll( + ` + SELECT id, bucket_name, op_id, op, source_table, source_key, table_name, row_id, checksum, target_op + FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id <= ? + ORDER BY op_id ASC + LIMIT ? + `, + [this.groupId, bucket, op, this.clearBatchLimit] + ) + ).map(rawBucketDataRow); + + let checksum = 0; + let lastOpId: bigint | null = null; + let targetOp: bigint | null = null; + let gotAnOp = false; + + for (const operation of operationRows) { + if (operation.op != 'MOVE' && operation.op != 'REMOVE' && operation.op != 'CLEAR') { + throw new Error(`Unexpected ${operation.op} operation at ${this.groupId}:${bucket}:${operation.opId}`); + } + + checksum = utils.addChecksums(checksum, Number(operation.checksum)); + lastOpId = operation.opId; + if (operation.op != 'CLEAR') { + gotAnOp = true; + } + if (operation.targetOp != null && (targetOp == null || operation.targetOp > targetOp)) { + targetOp = operation.targetOp; + } + } + + if (!gotAnOp || lastOpId == null) { + done = true; + return; + } + + this.logger.info(`Flushing CLEAR at ${lastOpId}`); + await this.executeRun( + ` + DELETE FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id <= ? + `, + [this.groupId, bucket, lastOpId] + ); + await this.executeRun( + ` + INSERT INTO bucket_data ( + id, + group_id, + bucket_name, + op_id, + op, + checksum, + target_op + ) VALUES (?, ?, ?, ?, 'CLEAR', ?, ?) + `, + [uuid.v4(), this.groupId, bucket, lastOpId, BigInt(checksum), targetOp] + ); + } + } + + private async executeAll(sql: string, params: unknown[]): Promise { + return this.dialect.db.$client.prepare(sql).all(...params) as T[]; + } + + private async executeRun(sql: string, params: unknown[]): Promise { + this.dialect.db.$client.prepare(sql).run(...params); + } +} + +function compactBucketRowKey(row: CompactionBucketDataRow): string { + return `${row.tableName}/${row.rowId}/${row.sourceTable}.${row.sourceKey == null ? '' : row.sourceKey.toString('base64')}`; +} + +function rawBucketDataRow(row: RawBucketDataRow): CompactionBucketDataRow { + return { + id: row.id, + bucketName: row.bucket_name, + opId: BigInt(row.op_id), + op: row.op, + sourceTable: row.source_table, + sourceKey: row.source_key == null ? null : Buffer.from(row.source_key), + tableName: row.table_name, + rowId: row.row_id, + checksum: BigInt(row.checksum), + targetOp: row.target_op == null ? null : BigInt(row.target_op) + }; +} + +function rawParameterRow(row: RawParameterCompactionRow): ParameterCompactionRow { + const sourceTable = row.source_table ?? row.sourceTable; + const sourceKey = row.source_key ?? row.sourceKey; + if (sourceTable == null || sourceKey == null) { + throw new Error('Expected parameter compaction row source columns'); + } + + return { + id: BigInt(row.id), + sourceTable, + sourceKey: Buffer.from(sourceKey), + lookup: Buffer.from(row.lookup), + bucketParameters: row.bucket_parameters ?? row.bucketParameters + }; +} + +function parameterKey(row: ParameterCompactionRow): string { + return `${row.lookup.toString('base64')}/${row.sourceTable}/${row.sourceKey.toString('base64')}`; +} + +function normalizedBucketParameters(value: unknown): string { + if (typeof value != 'string') { + return JSON.stringify(value ?? null); + } + + try { + const parsed = JSON.parse(value); + return typeof parsed == 'string' ? parsed : JSON.stringify(parsed); + } catch { + return value; + } +} + +function sameBucketParameters(left: unknown, right: unknown): boolean { + return normalizedBucketParameters(left) == normalizedBucketParameters(right); +} + +function isEmptyBucketParameters(value: unknown): boolean { + return normalizedBucketParameters(value) == '[]'; +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzlePersistedBatch.ts b/modules/module-drizzle-storage/src/storage/DrizzlePersistedBatch.ts new file mode 100644 index 000000000..2a23da15a --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzlePersistedBatch.ts @@ -0,0 +1,421 @@ +import type { Logger } from '@powersync/lib-services-framework'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import { inArray } from 'drizzle-orm'; +import * as uuid from 'uuid'; +import { bucketData, bucketParameters, type CurrentDataRow } from '../drivers/sqlite/schema.js'; +import type { DrizzleStorageTransaction } from '../drivers/sqlite/sqlite-config.js'; +import type { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +export interface DrizzlePersistedBatchOptions { + tx: DrizzleStorageTransaction; + dialect: DrizzleStorageDialect; + logger: Logger; + syncRules: sync_rules.HydratedSyncConfig; + replicationStreamId: number; + storeCurrentData: boolean; + skipExistingRows: boolean; + markRecordUnavailable: storage.BucketStorageMarkRecordUnavailable | undefined; +} + +export type CurrentBucket = { + bucket: string; + table: string; + id: string; +}; + +const MAX_ROW_SIZE = 15 * 1024 * 1024; + +/** + * Handles the writes for a single persisted operation chunk. + * + * `DrizzleBucketBatch` owns the public batch lifecycle, checkpointing, and + * listener hooks. This class owns the transactional write state for bucket + * data, parameter rows, and current data so large flushes can be split into + * smaller persisted chunks without keeping every tracked entity in one unit of + * work. + */ +export class DrizzlePersistedBatch { + private readonly currentDataById = new Map(); + + constructor(private readonly options: DrizzlePersistedBatchOptions) {} + + persistOperations(operations: storage.SaveOptions[], nextOpId: bigint): bigint { + this.loadCurrentData(operations); + for (const operation of operations) { + nextOpId = this.persistOperation(operation, nextOpId); + } + return nextOpId; + } + + persistBucketData(options: { + table: storage.SourceTable; + sourceKey: storage.ReplicaId; + existingBuckets: CurrentBucket[]; + evaluated: sync_rules.EvaluatedRow[]; + nextOpId: bigint; + }): bigint { + const remainingBuckets = new Map(options.existingBuckets.map((bucket) => [currentBucketKey(bucket), bucket])); + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + const deleteChecksum = utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)); + let nextOpId = options.nextOpId; + const values: (typeof bucketData.$inferInsert)[] = []; + + for (const row of options.evaluated) { + remainingBuckets.delete(currentBucketKey(row)); + const data = JSONBig.stringify(row.data); + const checksum = utils.hashData(row.table, row.id, data); + values.push({ + id: uuid.v4(), + groupId: this.options.replicationStreamId, + bucketName: row.bucket, + opId: nextOpId++, + op: 'PUT', + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + tableName: row.table, + rowId: row.id, + checksum: BigInt(checksum), + data, + targetOp: null + }); + } + + for (const bucket of remainingBuckets.values()) { + values.push({ + id: uuid.v4(), + groupId: this.options.replicationStreamId, + bucketName: bucket.bucket, + opId: nextOpId++, + op: 'REMOVE', + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + tableName: bucket.table, + rowId: bucket.id, + checksum: BigInt(deleteChecksum), + data: null, + targetOp: null + }); + } + + if (values.length > 0) { + this.options.tx.insert(this.options.dialect.tables.bucketData).values(values).run(); + } + + return nextOpId; + } + + persistParameterData(options: { + table: storage.SourceTable; + sourceKey: storage.ReplicaId; + existingLookups: Buffer[]; + evaluated: sync_rules.EvaluatedParameters[]; + nextOpId: bigint; + }): bigint { + const remainingLookups = new Map(options.existingLookups.map((lookup) => [lookup.toString('base64'), lookup])); + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + let nextOpId = options.nextOpId; + const values: (typeof bucketParameters.$inferInsert)[] = []; + + for (const row of options.evaluated) { + const lookup = storage.serializeLookupBuffer(row.lookup); + remainingLookups.delete(lookup.toString('base64')); + values.push({ + id: nextOpId++, + groupId: this.options.replicationStreamId, + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + lookup, + bucketParameters: JSONBig.stringify(row.bucketParameters) + }); + } + + for (const lookup of remainingLookups.values()) { + values.push({ + id: nextOpId++, + groupId: this.options.replicationStreamId, + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + lookup, + bucketParameters: '[]' + }); + } + + if (values.length > 0) { + this.options.tx.insert(this.options.dialect.tables.bucketParameters).values(values).run(); + } + + return nextOpId; + } + + private persistOperation(record: storage.SaveOptions, nextOpId: bigint): bigint { + const sourceTable = record.sourceTable; + const tableId = String(sourceTable.id); + const afterId = record.afterReplicaId ?? null; + const beforeId = record.beforeReplicaId ?? record.afterReplicaId; + const serializedBeforeId = storage.serializeReplicaId(beforeId); + const existingCurrentDataId = currentDataId(this.options.replicationStreamId, tableId, serializedBeforeId); + const existingCurrentData = this.currentDataById.get(existingCurrentDataId) ?? null; + + const storeCurrentData = this.options.storeCurrentData && sourceTable.storeCurrentData; + let existingBuckets = currentBuckets(existingCurrentData); + let existingLookups = currentLookups(existingCurrentData); + let after: sync_rules.ToastableSqliteRow | null | undefined = record.after; + + if (this.options.skipExistingRows) { + if (record.tag == storage.SaveOperationTag.INSERT) { + if (existingCurrentData != null) { + return nextOpId; + } + } else { + throw new ReplicationAssertionError(`${record.tag} not supported with skipExistingRows: true`); + } + } + + if (record.tag == storage.SaveOperationTag.UPDATE) { + if (existingCurrentData != null && storeCurrentData) { + after = storage.mergeToast(record.after, storage.deserializeBson(Buffer.from(existingCurrentData.data))); + } else if (existingCurrentData == null && storeCurrentData) { + this.options.markRecordUnavailable?.(record); + } + } + + if (beforeId != null && (afterId == null || !storage.replicaIdEquals(beforeId, afterId))) { + if (sourceTable.syncData) { + nextOpId = this.persistBucketData({ + table: sourceTable, + sourceKey: beforeId, + existingBuckets, + evaluated: [], + nextOpId + }); + existingBuckets = []; + } + + if (sourceTable.syncParameters) { + nextOpId = this.persistParameterData({ + table: sourceTable, + sourceKey: beforeId, + existingLookups, + evaluated: [], + nextOpId + }); + existingLookups = []; + } + } + + let newBuckets: CurrentBucket[] = []; + let newLookups: Buffer[] = []; + let afterData: Buffer | undefined; + let afterDataWasTruncated = false; + if (afterId != null && after != null && utils.isCompleteRow(storeCurrentData, after)) { + if (storeCurrentData) { + const prepared = this.serializeCurrentData(record, after); + after = prepared.after; + afterData = prepared.data; + afterDataWasTruncated = prepared.truncated; + } else { + afterData = storage.serializeBson({}); + } + + if (sourceTable.syncData) { + const { results: rawResults, errors } = this.options.syncRules.evaluateRowWithErrors({ + record: after as sync_rules.SqliteRow, + sourceTable: sourceTable.ref, + bucketDataSources: sourceTable.bucketDataSources + }); + const results = afterDataWasTruncated ? rawResults.filter(hasUsableObjectId) : rawResults; + for (const error of errors) { + this.options.logger.error( + `Failed to evaluate data query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + nextOpId = this.persistBucketData({ + table: sourceTable, + sourceKey: afterId, + existingBuckets, + evaluated: results, + nextOpId + }); + newBuckets = results.map((row) => ({ + bucket: row.bucket, + table: row.table, + id: row.id + })); + } + + if (sourceTable.syncParameters) { + const { results, errors } = this.options.syncRules.evaluateParameterRowWithErrors( + sourceTable.ref, + after as sync_rules.SqliteRow, + { + parameterLookupSources: sourceTable.parameterLookupSources + } + ); + for (const error of errors) { + this.options.logger.error( + `Failed to evaluate parameter query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + nextOpId = this.persistParameterData({ + table: sourceTable, + sourceKey: afterId, + existingLookups, + evaluated: results, + nextOpId + }); + newLookups = results.map((row) => storage.serializeLookupBuffer(row.lookup)); + } + } + + if (afterId != null && afterData != null) { + this.upsertCurrentData({ + tableId, + sourceKey: afterId, + buckets: newBuckets, + lookups: newLookups, + data: afterData, + pendingDelete: null + }); + } + + if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { + nextOpId = this.deleteCurrentData(tableId, beforeId, nextOpId); + } + + return nextOpId; + } + + private loadCurrentData(operations: storage.SaveOptions[]): void { + const ids = new Set(); + for (const operation of operations) { + const tableId = String(operation.sourceTable.id); + const beforeId = operation.beforeReplicaId ?? operation.afterReplicaId; + ids.add(currentDataId(this.options.replicationStreamId, tableId, storage.serializeReplicaId(beforeId))); + + const afterId = operation.afterReplicaId ?? null; + if (afterId != null) { + ids.add(currentDataId(this.options.replicationStreamId, tableId, storage.serializeReplicaId(afterId))); + } + } + + const rows = + ids.size == 0 + ? [] + : this.options.tx + .select() + .from(this.options.dialect.tables.currentData) + .where(inArray(this.options.dialect.tables.currentData.id, [...ids])) + .all(); + for (const row of rows) { + this.currentDataById.set(row.id, row); + } + } + + private serializeCurrentData( + record: storage.SaveOptions, + after: sync_rules.ToastableSqliteRow + ): { after: sync_rules.ToastableSqliteRow; data: Buffer; truncated: boolean } { + try { + const serialized = storage.serializeBson(after); + if (serialized.byteLength > MAX_ROW_SIZE) { + throw new Error(`Row too large: ${serialized.byteLength}`); + } + return { after, data: serialized, truncated: false }; + } catch (e) { + const error = e instanceof Error ? e : new Error(String(e)); + this.options.logger.warn( + `Data too big on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.message}` + ); + + // Keep the current_data row present, but drop field values. This mirrors + // the Postgres storage behavior for oversized BSON payloads and allows + // future TOAST-style updates to be marked unavailable instead of crashing + // the replication batch. + const emptyValues = Object.fromEntries(Object.keys(after).map((key) => [key, undefined])); + return { after: emptyValues, data: storage.serializeBson(emptyValues), truncated: true }; + } + } + + private upsertCurrentData(options: { + tableId: string; + sourceKey: storage.ReplicaId; + buckets: CurrentBucket[]; + lookups: Buffer[]; + data: Buffer; + pendingDelete: bigint | null; + }): void { + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + const id = currentDataId(this.options.replicationStreamId, options.tableId, serializedSourceKey); + const payload = { + id, + groupId: this.options.replicationStreamId, + sourceTable: options.tableId, + sourceKey: serializedSourceKey, + buckets: options.buckets, + lookups: options.lookups.map((lookup) => lookup.toString('hex')), + data: options.data, + pendingDelete: options.pendingDelete + }; + this.options.tx + .insert(this.options.dialect.tables.currentData) + .values(payload) + .onConflictDoUpdate({ + target: this.options.dialect.tables.currentData.id, + set: payload + }) + .run(); + this.currentDataById.set(id, payload); + } + + private deleteCurrentData(tableId: string, sourceKey: storage.ReplicaId, nextOpId: bigint): bigint { + const serializedSourceKey = storage.serializeReplicaId(sourceKey); + const id = currentDataId(this.options.replicationStreamId, tableId, serializedSourceKey); + const payload = { + id, + groupId: this.options.replicationStreamId, + sourceTable: tableId, + sourceKey: serializedSourceKey, + buckets: [], + lookups: [], + data: storage.serializeBson({}), + pendingDelete: nextOpId + }; + this.options.tx + .insert(this.options.dialect.tables.currentData) + .values(payload) + .onConflictDoUpdate({ target: this.options.dialect.tables.currentData.id, set: payload }) + .run(); + this.currentDataById.set(id, payload); + return nextOpId + 1n; + } +} + +export function currentBuckets(row: CurrentDataRow | null): CurrentBucket[] { + return Array.isArray(row?.buckets) ? (row.buckets as CurrentBucket[]) : []; +} + +export function currentLookups(row: CurrentDataRow | null): Buffer[] { + return Array.isArray(row?.lookups) ? (row.lookups as string[]).map((lookup) => Buffer.from(lookup, 'hex')) : []; +} + +function currentDataId(groupId: number, sourceTable: string, sourceKey: Buffer): string { + return `${groupId}:${sourceTable}:${sourceKey.toString('hex')}`; +} + +function currentBucketKey(bucket: CurrentBucket | sync_rules.EvaluatedRow): string { + return `${bucket.bucket}/${bucket.table}/${bucket.id}`; +} + +function hasUsableObjectId(row: sync_rules.EvaluatedRow): boolean { + return row.id !== '' || row.data.id != null; +} + +function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string { + if (storage.isUUID(id)) { + return `${tableId}/${id.toHexString()}`; + } + return uuid.v5(storage.serializeBson({ table: tableId, id }), utils.ID_NAMESPACE); +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts b/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts new file mode 100644 index 000000000..21070459a --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzlePersistedReplicationStream.ts @@ -0,0 +1,76 @@ +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { eq } from 'drizzle-orm'; +import type { SyncRulesRow } from '../drivers/sqlite/schema.js'; +import type { DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +export class DrizzlePersistedSyncConfigContent extends storage.PersistedSyncConfigContent { + constructor( + private readonly dialect: DrizzleStorageDialect, + row: SyncRulesRow + ) { + super({ + replicationStreamId: row.id, + sync_rules_content: row.content, + compiled_plan: row.syncPlan, + replicationStreamName: row.slotName, + storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION, + syncConfigId: String(row.id), + syncConfigState: row.state as storage.SyncRuleState + }); + } + + async getSyncConfigStatus(): Promise { + const { db, tables } = this.dialect; + const row = db.select().from(tables.syncRules).where(eq(tables.syncRules.id, this.replicationStreamId)).get(); + return row == null ? null : syncConfigStatusFromRow(row); + } +} + +export class DrizzlePersistedReplicationStream extends storage.PersistedReplicationStream { + current_lock: storage.ReplicationLock | null = null; + readonly syncConfigContent: readonly DrizzlePersistedSyncConfigContent[]; + + constructor( + private readonly dialect: DrizzleStorageDialect, + private readonly row: SyncRulesRow + ) { + super({ + replicationStreamId: row.id, + replicationStreamName: row.slotName, + state: row.state, + storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION + }); + this.syncConfigContent = [new DrizzlePersistedSyncConfigContent(this.dialect, this.row)]; + } + + parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet { + return this.syncConfigContent[0].parsed(options); + } + + async lock(): Promise { + if (this.current_lock != null) { + throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by this process.`); + } + return (this.current_lock = { + sync_rules_id: this.replicationStreamId, + release: async () => { + this.current_lock = null; + } + }); + } +} + +export function syncConfigStatusFromRow(row: SyncRulesRow): storage.PersistedSyncConfigStatus { + return { + id: String(row.id), + replicationStreamId: row.id, + state: row.state, + snapshot_done: row.snapshotDone, + last_checkpoint_lsn: row.lastCheckpointLsn, + last_fatal_error: row.lastFatalError, + last_fatal_error_ts: row.lastFatalErrorTs, + last_keepalive_ts: row.lastKeepaliveTs, + last_checkpoint_ts: row.lastCheckpointTs + }; +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleReportStorage.ts b/modules/module-drizzle-storage/src/storage/DrizzleReportStorage.ts new file mode 100644 index 000000000..1cb4de175 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleReportStorage.ts @@ -0,0 +1,34 @@ +import { storage } from '@powersync/service-core'; +import { event_types } from '@powersync/service-types'; + +export class DrizzleReportStorage implements storage.ReportStorage { + async [Symbol.asyncDispose](): Promise { + // Report storage is intentionally a no-op in the initial Drizzle storage slice. + } + + async reportClientConnection(_data: event_types.ClientConnectionBucketData): Promise {} + + async reportClientDisconnection(_data: event_types.ClientDisconnectionEventData): Promise {} + + async getConnectedClients(): Promise { + return { users: [], sdks: [] } as unknown as event_types.ClientConnectionReportResponse; + } + + async getClientConnectionReports( + _data: event_types.ClientConnectionReportRequest + ): Promise { + return { users: [], sdks: [] } as unknown as event_types.ClientConnectionReportResponse; + } + + async getGeneralClientConnectionAnalytics( + _data: event_types.ClientConnectionAnalyticsRequest + ): Promise> { + return { + items: [], + count: 0, + more: false + } as unknown as event_types.PaginatedResponse; + } + + async deleteOldConnectionData(_data: event_types.DeleteOldConnectionData): Promise {} +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts b/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts new file mode 100644 index 000000000..7bab0517f --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleStorageDialect.ts @@ -0,0 +1,57 @@ +import type { BucketDataRequest } from '@powersync/service-core'; +import type { BucketDataRow, sqliteSchema } from '../drivers/sqlite/schema.js'; +import type { DrizzleStorageDatabase, DrizzleStorageTransaction } from '../drivers/sqlite/sqlite-config.js'; + +export interface DrizzleStorageDialect { + readonly type: string; + readonly db: DrizzleStorageDatabase; + readonly tables: typeof sqliteSchema; + transaction(callback: (tx: DrizzleStorageTransaction) => T): T; + streamBucketDataRows(options: DrizzleBucketDataStreamOptions): AsyncIterable; + createCheckpointWatcher(): DrizzleCheckpointWatcher; +} + +export type BucketDataReadRow = Omit; + +export interface DrizzleBucketDataStreamOptions { + readonly db?: DrizzleStorageDatabase; + readonly groupId: number; + readonly checkpoint: bigint; + readonly dataBuckets: BucketDataRequest[]; + readonly limit: number; +} + +export interface DrizzleCheckpointWatcher { + notify(): void; + watch(signal: AbortSignal): AsyncIterable; +} + +export class InProcessDrizzleCheckpointWatcher implements DrizzleCheckpointWatcher { + private readonly listeners = new Set<() => void>(); + + notify(): void { + for (const listener of this.listeners) { + listener(); + } + } + + async *watch(signal: AbortSignal): AsyncIterable { + while (!signal.aborted) { + yield await new Promise((resolve) => { + const listener = () => { + this.listeners.delete(listener); + resolve(); + }; + this.listeners.add(listener); + signal.addEventListener( + 'abort', + () => { + this.listeners.delete(listener); + resolve(); + }, + { once: true } + ); + }); + } + } +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleStorageProvider.ts b/modules/module-drizzle-storage/src/storage/DrizzleStorageProvider.ts new file mode 100644 index 000000000..0a4e8abc7 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleStorageProvider.ts @@ -0,0 +1,67 @@ +import { logger } from '@powersync/lib-services-framework'; +import { storage, system } from '@powersync/service-core'; +import { createSqliteDrizzleStorageFactory } from '../drivers/sqlite/SqliteDrizzleStorageFactory.js'; +import { + DRIZZLE_SQLITE_STORAGE_TYPE, + DrizzleSqliteStorageConfig, + isDrizzleStorageConfig, + normalizeDrizzleSqliteStorageConfig +} from '../types/types.js'; +import { DrizzleReportStorage } from './DrizzleReportStorage.js'; + +export class DrizzleStorageProvider implements storage.StorageProvider { + get type(): typeof DRIZZLE_SQLITE_STORAGE_TYPE { + return DRIZZLE_SQLITE_STORAGE_TYPE; + } + + async getStorage(options: storage.GetStorageOptions): Promise { + const storageConfig = options.resolvedConfig.storage; + if (!isDrizzleStorageConfig(storageConfig)) { + throw new Error(`Cannot create Drizzle storage with provided config ${storageConfig.type}`); + } + assertSqliteServiceMode(options.serviceMode); + const normalizedConfig = normalizeDrizzleSqliteStorageConfig(DrizzleSqliteStorageConfig.decode(storageConfig)); + const factory = createSqliteDrizzleStorageFactory({ + config: normalizedConfig, + slotNamePrefix: options.resolvedConfig.slot_name_prefix + }); + + return { + reportStorage: new DrizzleReportStorage(), + storage: factory, + shutDown: async () => factory[Symbol.asyncDispose](), + tearDown: async () => { + logger.info(`Tearing down Drizzle SQLite storage: ${normalizedConfig.filename}...`); + for (const table of [ + 'write_checkpoints', + 'bucket_parameters', + 'current_data', + 'bucket_data', + 'source_tables', + 'sync_rules', + 'instance' + ]) { + factory.runtime.client.exec(`DROP TABLE IF EXISTS \`${table}\``); + } + await factory[Symbol.asyncDispose](); + return true; + } + }; + } +} + +const SQLITE_ALLOWED_COMMAND_MODES = new Set([ + system.ServiceContextMode.COMPACT, + system.ServiceContextMode.TEARDOWN, + system.ServiceContextMode.TEST_CONNECTION +]); + +function assertSqliteServiceMode(serviceMode: string): void { + if (serviceMode == system.ServiceContextMode.UNIFIED || SQLITE_ALLOWED_COMMAND_MODES.has(serviceMode)) { + return; + } + throw new Error( + `Drizzle SQLite storage only supports the unified service runner. ` + + `SQLite checkpoint notifications are process-local, so split runners cannot safely share this storage.` + ); +} diff --git a/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts b/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts new file mode 100644 index 000000000..5b2087df4 --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/DrizzleSyncRulesStorage.ts @@ -0,0 +1,647 @@ +import { BaseObserver, DO_NOT_LOG, errors, Logger } from '@powersync/lib-services-framework'; +import { + BucketChecksumRequest, + BucketDataBatchOptions, + BucketDataRequest, + CHECKPOINT_INVALIDATE_ALL, + CheckpointChanges, + GetCheckpointChangesOptions, + PopulateChecksumCacheOptions, + PopulateChecksumCacheResults, + ReplicationCheckpoint, + storage, + StorageCheckpointUpdate, + SyncBucketDataChunk, + utils, + WatchWriteCheckpointOptions +} from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import { and, count, desc, eq, gt, isNull, lte, sum } from 'drizzle-orm'; +import * as uuid from 'uuid'; +import type { BucketDataRow } from '../drivers/sqlite/schema.js'; +import { DrizzleBucketBatch } from './DrizzleBucketBatch.js'; +import { DrizzleBucketStorageFactory } from './DrizzleBucketStorageFactory.js'; +import { DrizzleCompactor } from './DrizzleCompactor.js'; +import { BucketDataReadRow, DrizzleStorageDialect } from './DrizzleStorageDialect.js'; + +export interface DrizzleSyncRulesStorageOptions { + factory: DrizzleBucketStorageFactory; + dialect: DrizzleStorageDialect; + replicationStream: storage.PersistedReplicationStream; +} + +export class DrizzleSyncRulesStorage + extends BaseObserver + implements storage.SyncRulesBucketStorage +{ + [DO_NOT_LOG] = true; + + readonly replicationStreamId: number; + readonly replicationStreamName: string; + readonly storageConfig: storage.StorageVersionConfig; + readonly factory: DrizzleBucketStorageFactory; + readonly logger: Logger; + + private readonly parsedSyncConfigSets = new Map(); + + private writeCheckpointModeValue = storage.WriteCheckpointMode.MANAGED; + private checksumCacheValue: storage.ChecksumCache | undefined; + + constructor(private readonly options: DrizzleSyncRulesStorageOptions) { + super(); + this.replicationStreamId = options.replicationStream.replicationStreamId; + this.replicationStreamName = options.replicationStream.replicationStreamName; + this.storageConfig = options.replicationStream.getStorageConfig(); + this.factory = options.factory; + this.logger = options.replicationStream.logger; + } + + get writeCheckpointMode(): storage.WriteCheckpointMode { + return this.writeCheckpointModeValue; + } + + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { + this.writeCheckpointModeValue = mode; + } + + private get checksumCache(): storage.ChecksumCache { + this.checksumCacheValue ??= new storage.ChecksumCache({ + fetchChecksums: (batch) => this.getChecksumsInternal(batch) + }); + return this.checksumCacheValue; + } + + async createManagedWriteCheckpoints( + checkpoints: storage.ManagedWriteCheckpointOptions[] + ): Promise { + if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) { + throw new errors.ValidationError( + `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` + ); + } + + const uniqueCheckpoints = storage.uniqueManagedWriteCheckpoints(checkpoints); + if (uniqueCheckpoints.length == 0) { + return { writeCheckpoints: new Map(), shouldAdvance: false }; + } + const table = this.options.dialect.tables.writeCheckpoints; + const writeCheckpoints = this.options.dialect.transaction((tx) => { + const result = new Map(); + for (const checkpoint of uniqueCheckpoints) { + const latest = tx + .select() + .from(table) + .where(and(eq(table.userId, checkpoint.user_id), isNull(table.syncRulesId))) + .orderBy(desc(table.checkpoint)) + .limit(1) + .get(); + + const requestedCheckpoint = checkpoint.checkpoint_request_id; + if (requestedCheckpoint != null && latest != null && requestedCheckpoint <= latest.checkpoint) { + if (requestedCheckpoint == latest.checkpoint) { + tx.update(table).set({ checkpointRequestedAt: new Date() }).where(eq(table.id, latest.id)).run(); + } + result.set(checkpoint.user_id, latest.checkpoint); + continue; + } + + if (requestedCheckpoint != null && latest != null) { + // A newer client-supplied id replaces the managed mapping. Keeping an + // older row would allow that old id to be acknowledged while the new + // request's source head is still pending. + tx.delete(table) + .where(and(eq(table.userId, checkpoint.user_id), isNull(table.syncRulesId))) + .run(); + } + + const value = requestedCheckpoint ?? (latest?.checkpoint ?? 0n) + 1n; + tx.insert(table) + .values({ + id: uuid.v4(), + syncRulesId: null, + userId: checkpoint.user_id, + checkpoint: value, + heads: checkpoint.heads, + checkpointRequestedAt: requestedCheckpoint == null ? null : new Date(), + createdAt: new Date() + }) + .run(); + result.set(checkpoint.user_id, value); + } + return result; + }); + + this.factory.checkpointWatcher.notify(); + // This storage does not track whether each managed checkpoint has already + // been processed, so conservatively force the source marker for every + // matched request. This also makes stale retries recover a lost marker. + return { writeCheckpoints, shouldAdvance: true }; + } + + async lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { + switch (this.writeCheckpointMode) { + case storage.WriteCheckpointMode.CUSTOM: + return this.lastCustomWriteCheckpoint({ + user_id: filters.user_id, + sync_rules_id: this.replicationStreamId + }); + case storage.WriteCheckpointMode.MANAGED: + if (!('heads' in filters)) { + throw new errors.ValidationError(`Replication HEAD is required for managed Write Checkpoint filtering`); + } + return this.lastManagedWriteCheckpoint(filters); + } + } + + async createWriter(options: storage.CreateWriterOptions): Promise { + const { db, tables } = this.options.dialect; + const syncRules = db.select().from(tables.syncRules).where(eq(tables.syncRules.id, this.replicationStreamId)).get(); + + const checkpointLsn = syncRules?.lastCheckpointLsn ?? null; + const writer = new DrizzleBucketBatch({ + factory: this.factory, + dialect: this.options.dialect, + logger: options.logger ?? this.logger, + syncRules: this.getParsedSyncRules(options), + replicationStreamId: this.replicationStreamId, + replicationStreamName: this.replicationStreamName, + lastCheckpointLsn: checkpointLsn, + keepaliveOp: syncRules?.keepaliveOp ?? null, + resumeFromLsn: utils.maxLsn(syncRules?.snapshotLsn, checkpointLsn), + storeCurrentData: options.storeCurrentData, + skipExistingRows: options.skipExistingRows ?? false, + markRecordUnavailable: options.markRecordUnavailable, + hooks: options.hooks + }); + this.iterateListeners((cb) => cb.batchStarted?.(writer)); + return writer; + } + + async startBatch( + options: storage.CreateWriterOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + await using writer = await this.createWriter(options); + await callback(writer); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + } + + getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet { + let parsed = this.parsedSyncConfigSets.get(options.defaultSchema); + if (parsed == null) { + parsed = this.options.replicationStream.parsed(options); + this.parsedSyncConfigSets.set(options.defaultSchema, parsed); + } + return parsed; + } + + getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig { + return this.getParsedSyncConfigSet(options).hydratedSyncConfig; + } + + async terminate(options?: storage.TerminateOptions): Promise { + if (!options || options.clearStorage) { + await this.clear(options); + } + + const { db, tables } = this.options.dialect; + db.update(tables.syncRules) + .set({ + state: storage.SyncRuleState.TERMINATED, + snapshotDone: false + }) + .where(eq(tables.syncRules.id, this.replicationStreamId)) + .run(); + this.factory.checkpointWatcher.notify(); + } + + async getStatus(): Promise { + const { db, tables } = this.options.dialect; + const row = db.select().from(tables.syncRules).where(eq(tables.syncRules.id, this.replicationStreamId)).get(); + + if (row == null) { + throw new Error('Cannot find replication stream status'); + } + return { + snapshotDone: row.snapshotDone && row.lastCheckpointLsn != null, + resumeLsn: utils.maxLsn(row.snapshotLsn, row.lastCheckpointLsn) + }; + } + + async clear(_options?: storage.ClearStorageOptions): Promise { + const tables = this.options.dialect.tables; + this.options.dialect.transaction((tx) => { + tx.update(tables.syncRules) + .set({ + snapshotDone: false, + lastCheckpointLsn: null, + lastCheckpoint: null, + noCheckpointBefore: null + }) + .where(eq(tables.syncRules.id, this.replicationStreamId)) + .run(); + tx.delete(tables.bucketData).where(eq(tables.bucketData.groupId, this.replicationStreamId)).run(); + tx.delete(tables.bucketParameters).where(eq(tables.bucketParameters.groupId, this.replicationStreamId)).run(); + tx.delete(tables.currentData).where(eq(tables.currentData.groupId, this.replicationStreamId)).run(); + tx.delete(tables.sourceTables).where(eq(tables.sourceTables.groupId, this.replicationStreamId)).run(); + }); + + this.clearChecksumCache(); + this.factory.checkpointWatcher.notify(); + } + + async reportError(e: any): Promise { + const { db, tables } = this.options.dialect; + db.update(tables.syncRules) + .set({ + lastFatalError: String(e.message ?? 'Replication failure'), + lastFatalErrorTs: new Date() + }) + .where(eq(tables.syncRules.id, this.replicationStreamId)) + .run(); + } + + async compact(options?: storage.CompactOptions): Promise { + let maxOpId = options?.maxOpId; + if (maxOpId == null) { + const checkpoint = await this.getCheckpoint(); + maxOpId = checkpoint.checkpoint; + } + + const compactor = new DrizzleCompactor(this.options.dialect, this.replicationStreamId, { + ...options, + maxOpId, + logger: this.logger + }); + await compactor.compact(); + + if (options?.compactParameterData) { + await compactor.compactParameterData(options); + } + + // Compaction can replace operations at an already-cached checkpoint with a + // CLEAR operation, invalidating any incremental checksum based on it. + this.clearChecksumCache(); + } + + async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise { + return { buckets: 0 }; + } + + async getCheckpoint(): Promise { + const { db, tables } = this.options.dialect; + const row = db.select().from(tables.syncRules).where(eq(tables.syncRules.id, this.replicationStreamId)).get(); + + return { + checkpoint: row?.lastCheckpoint ?? 0n, + lsn: row?.lastCheckpointLsn ?? null, + getParameterSets: (lookups, limit) => this.getParameterSets(row?.lastCheckpoint ?? 0n, lookups, limit) + }; + } + + async getCheckpointChanges(_options: GetCheckpointChangesOptions): Promise { + return CHECKPOINT_INVALIDATE_ALL; + } + + async *watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable { + let lastCheckpoint: bigint | null = null; + let lastCheckpointLsn: string | null = null; + let lastWriteCheckpoint: bigint | null = null; + const { signal, user_id } = options; + + const watcher = this.factory.checkpointWatcher.watch(signal)[Symbol.asyncIterator](); + let nextNotification: Promise> | null = null; + let readImmediately = true; + + try { + while (!signal.aborted) { + if (!readImmediately) { + nextNotification ??= watcher.next(); + const result = await nextNotification; + nextNotification = null; + if (result.done) { + return; + } + } + readImmediately = false; + + if (signal.aborted) { + return; + } + + const base = await this.getCheckpoint(); + const currentWriteCheckpoint = await this.lastWriteCheckpoint({ + user_id, + heads: base.lsn == null ? {} : { '1': base.lsn } + }); + + if ( + currentWriteCheckpoint == lastWriteCheckpoint && + base.checkpoint == lastCheckpoint && + base.lsn == lastCheckpointLsn + ) { + continue; + } + + lastWriteCheckpoint = currentWriteCheckpoint; + lastCheckpoint = base.checkpoint; + lastCheckpointLsn = base.lsn; + nextNotification = watcher.next(); + + yield { + base, + writeCheckpoint: currentWriteCheckpoint, + update: CHECKPOINT_INVALIDATE_ALL + }; + } + } finally { + await watcher.return?.(); + } + } + + async *getBucketDataBatch( + checkpoint: ReplicationCheckpoint, + dataBuckets: BucketDataRequest[], + options?: BucketDataBatchOptions + ): AsyncIterable { + if (dataBuckets.length == 0) { + return; + } + + const batchRowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + const startOpByBucket = new Map(dataBuckets.map((request) => [request.bucket, request.start])); + const rows = this.options.dialect.streamBucketDataRows({ + groupId: this.replicationStreamId, + checkpoint: checkpoint.checkpoint, + dataBuckets, + limit: batchRowLimit + }); + + let chunkSizeBytes = 0; + let currentChunk: utils.SyncBucketData | null = null; + let targetOp: bigint | null = null; + let batchRowCount = 0; + + for await (const row of rows) { + const rowSizeBytes = row.data?.length ?? 0; + const sizeExceeded = + chunkSizeBytes >= chunkSizeLimitBytes || + ((currentChunk?.data.length ?? 0) > 0 && chunkSizeBytes + rowSizeBytes > chunkSizeLimitBytes) || + (currentChunk?.data.length ?? 0) >= batchRowLimit; + + if (currentChunk == null || currentChunk.bucket != row.bucketName || sizeExceeded) { + let start: string | undefined = undefined; + if (currentChunk != null) { + if (currentChunk.bucket == row.bucketName) { + currentChunk.has_more = true; + start = currentChunk.next_after; + } + + const yieldChunk = currentChunk; + currentChunk = null; + chunkSizeBytes = 0; + yield { chunkData: yieldChunk, targetOp }; + targetOp = null; + if (batchRowCount >= batchRowLimit) { + break; + } + } + + if (start == null) { + const startOpId = startOpByBucket.get(row.bucketName); + if (startOpId == null) { + throw new Error(`data for unexpected bucket: ${row.bucketName}`); + } + start = utils.internalToExternalOpId(startOpId); + } + currentChunk = { + bucket: row.bucketName, + after: start, + has_more: false, + data: [], + next_after: start + }; + } + + const entry = bucketDataRowToOpEntry(row); + currentChunk.data.push(entry); + currentChunk.next_after = entry.op_id; + if (row.targetOp != null && (targetOp == null || row.targetOp > targetOp)) { + targetOp = row.targetOp; + } + + chunkSizeBytes += rowSizeBytes; + batchRowCount++; + } + + if (currentChunk != null) { + currentChunk.has_more = batchRowCount >= batchRowLimit; + yield { chunkData: currentChunk, targetOp }; + } + } + + async getChecksums(checkpoint: ReplicationCheckpoint, buckets: BucketChecksumRequest[]): Promise { + return this.checksumCache.getChecksumMap(checkpoint.checkpoint, buckets); + } + + clearChecksumCache(): void { + this.checksumCacheValue?.clear(); + } + + private async getChecksumsInternal(batch: storage.FetchPartialBucketChecksum[]): Promise { + const result: storage.PartialChecksumMap = new Map(); + if (batch.length == 0) { + return result; + } + + const { db, tables } = this.options.dialect; + + for (const request of batch) { + const rangeFilter = and( + eq(tables.bucketData.groupId, this.replicationStreamId), + eq(tables.bucketData.bucketName, request.bucket), + gt(tables.bucketData.opId, request.start ?? 0n), + lte(tables.bucketData.opId, request.end) + ); + const aggregate = db + .select({ + checksum: sum(tables.bucketData.checksum), + count: count() + }) + .from(tables.bucketData) + .where(rangeFilter) + .get(); + const checksum = Number(BigInt(aggregate?.checksum ?? 0) & 0xffffffffn) & 0xffffffff; + const rowCount = aggregate?.count ?? 0; + + // A CLEAR only changes how a partial result is combined with an existing + // cached checksum. Full requests are already combined with an empty base. + const hasClearOperation = + request.start != null && + db + .select({ opId: tables.bucketData.opId }) + .from(tables.bucketData) + .where(and(rangeFilter, eq(tables.bucketData.op, 'CLEAR'))) + .limit(1) + .get() != null; + + result.set( + request.bucket, + hasClearOperation + ? { bucket: request.bucket, checksum, count: rowCount } + : { bucket: request.bucket, partialChecksum: checksum, partialCount: rowCount } + ); + } + + return result; + } + + private async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters): Promise { + const { db, tables } = this.options.dialect; + const row = db + .select() + .from(tables.writeCheckpoints) + .where( + and( + eq(tables.writeCheckpoints.userId, filters.user_id), + filters.sync_rules_id == null + ? isNull(tables.writeCheckpoints.syncRulesId) + : eq(tables.writeCheckpoints.syncRulesId, filters.sync_rules_id) + ) + ) + .orderBy(desc(tables.writeCheckpoints.checkpoint)) + .limit(1) + .get(); + return row?.checkpoint ?? null; + } + + private async getParameterSets( + checkpoint: bigint, + lookups: sync_rules.ScopedParameterLookup[], + limit: number + ): Promise { + const resultsByLookup = new Map(); + let totalRows = 0; + + for (const lookup of lookups) { + const serializedLookup = storage.serializeLookupBuffer(lookup); + const { db, tables } = this.options.dialect; + const rows = db + .select() + .from(tables.bucketParameters) + .where( + and( + eq(tables.bucketParameters.groupId, this.replicationStreamId), + eq(tables.bucketParameters.lookup, serializedLookup), + lte(tables.bucketParameters.id, checkpoint) + ) + ) + .orderBy(desc(tables.bucketParameters.id)) + .all(); + + const latestBySource = new Map(); + for (const row of rows) { + const key = `${row.sourceTable}:${Buffer.from(row.sourceKey).toString('hex')}`; + if (!latestBySource.has(key)) { + latestBySource.set(key, row); + } + } + + for (const row of latestBySource.values()) { + const parameterRows = parseBucketParameters(row.bucketParameters); + if (parameterRows.length == 0) { + continue; + } + totalRows += parameterRows.length; + if (totalRows > limit) { + throw new storage.ParameterSetLimitExceededError(limit); + } + const existing = resultsByLookup.get(lookup); + if (existing != null) { + existing.push(...parameterRows); + } else { + resultsByLookup.set(lookup, parameterRows); + } + } + } + + const results: sync_rules.ParameterLookupRows[] = []; + resultsByLookup.forEach((rows, lookup) => results.push({ lookup, rows })); + return results; + } + + private async lastManagedWriteCheckpoint(filters: storage.ManagedWriteCheckpointFilters): Promise { + const lsn = filters.heads['1']; + if (lsn == null) { + return null; + } + + const { db, tables } = this.options.dialect; + // Luckily, we usually only have one record per user here + const rows = db + .select({ + checkpoint: tables.writeCheckpoints.checkpoint, + heads: tables.writeCheckpoints.heads + }) + .from(tables.writeCheckpoints) + .where(and(eq(tables.writeCheckpoints.userId, filters.user_id), isNull(tables.writeCheckpoints.syncRulesId))) + .orderBy(desc(tables.writeCheckpoints.checkpoint)) + .all(); + + return ( + rows.find((row) => { + const rowHead = getPrimaryReplicationHead(row.heads); + return rowHead != null && rowHead <= lsn; + })?.checkpoint ?? null + ); + } +} + +function getPrimaryReplicationHead(heads: unknown): string | null { + if (heads == null || typeof heads != 'object' || Array.isArray(heads)) { + return null; + } + + const head = (heads as Record)['1']; + return typeof head == 'string' ? head : null; +} + +function parseBucketParameters(value: unknown): sync_rules.SqliteJsonRow[] { + if (typeof value == 'string') { + return JSONBig.parse(value) as sync_rules.SqliteJsonRow[]; + } + return Array.isArray(value) ? (value as sync_rules.SqliteJsonRow[]) : []; +} + +function bucketDataRowToOpEntry(row: BucketDataRow | BucketDataReadRow): utils.OplogEntry { + if (row.op == 'PUT' || row.op == 'REMOVE') { + return { + op_id: utils.internalToExternalOpId(row.opId), + op: row.op, + object_type: row.tableName ?? undefined, + object_id: row.rowId ?? undefined, + checksum: Number(row.checksum), + subkey: + row.sourceTable != null && row.sourceKey != null + ? replicaIdToSubkey(row.sourceTable, storage.deserializeReplicaId(Buffer.from(row.sourceKey))) + : undefined, + data: row.op == 'REMOVE' ? null : (row.data ?? undefined) + }; + } + + return { + op_id: utils.internalToExternalOpId(row.opId), + op: row.op as 'CLEAR' | 'MOVE', + checksum: Number(row.checksum) + }; +} + +function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string { + if (storage.isUUID(id)) { + return `${tableId}/${id.toHexString()}`; + } + return uuid.v5(storage.serializeBson({ table: tableId, id }), utils.ID_NAMESPACE); +} diff --git a/modules/module-drizzle-storage/src/storage/storage-index.ts b/modules/module-drizzle-storage/src/storage/storage-index.ts new file mode 100644 index 000000000..5f455092a --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/storage-index.ts @@ -0,0 +1,5 @@ +export * from './DrizzleBucketBatch.js'; +export * from './DrizzleBucketStorageFactory.js'; +export * from './DrizzlePersistedReplicationStream.js'; +export * from './DrizzleStorageProvider.js'; +export * from './DrizzleSyncRulesStorage.js'; diff --git a/modules/module-drizzle-storage/src/storage/unsupported.ts b/modules/module-drizzle-storage/src/storage/unsupported.ts new file mode 100644 index 000000000..7deb9325a --- /dev/null +++ b/modules/module-drizzle-storage/src/storage/unsupported.ts @@ -0,0 +1,3 @@ +export function unsupportedDrizzleStorageFeature(feature: string): never { + throw new Error(`Drizzle bucket storage does not implement ${feature} yet.`); +} diff --git a/modules/module-drizzle-storage/src/types/records.ts b/modules/module-drizzle-storage/src/types/records.ts new file mode 100644 index 000000000..e5475c952 --- /dev/null +++ b/modules/module-drizzle-storage/src/types/records.ts @@ -0,0 +1,99 @@ +import type { storage } from '@powersync/service-core'; +import type { + BucketDataRow, + BucketParametersRow, + CurrentDataRow, + SourceTableRow, + SyncRulesRow, + WriteCheckpointRow +} from '../drivers/sqlite/schema.js'; + +export interface StorageBucketDataRecord { + id: string; + groupId: number; + bucketName: string; + opId: bigint; + op: string; + sourceTable: string | null; + sourceKey: Buffer | null; + tableName: string | null; + rowId: string | null; + checksum: bigint; + data: string | null; + targetOp: bigint | null; +} + +export interface StorageBucketParametersRecord { + id: bigint; + groupId: number; + sourceTable: string; + sourceKey: Buffer; + lookup: Buffer; + bucketParameters: unknown; +} + +export interface StorageCurrentDataRecord { + id: string; + groupId: number; + sourceTable: string; + sourceKey: Buffer; + buckets: { bucket: string; table: string; id: string }[]; + lookups: string[]; + data: Buffer; + pendingDelete: bigint | null; +} + +export interface StorageSourceTableRecord { + id: string; + groupId: number; + connectionId: number; + relationId: unknown; + schemaName: string; + tableName: string; + replicaIdColumns: unknown; + snapshotDone: boolean; + snapshotTotalEstimatedCount: bigint | null; + snapshotReplicatedCount: bigint | null; + snapshotLastKey: Buffer | null; +} + +export interface StorageSyncRulesRecord { + id: number; + state: storage.SyncRuleState; + snapshotDone: boolean; + snapshotLsn: string | null; + lastCheckpoint: bigint | null; + lastCheckpointLsn: string | null; + noCheckpointBefore: string | null; + slotName: string; + lastCheckpointTs: Date | null; + lastKeepaliveTs: Date | null; + lastFatalError: string | null; + lastFatalErrorTs: Date | null; + keepaliveOp: bigint | null; + storageVersion: number | null; + content: string; + syncPlan: storage.SerializedSyncPlan | null; +} + +export interface StorageWriteCheckpointRecord { + id: string; + syncRulesId: number | null; + userId: string; + checkpoint: bigint; + heads: Record | null; + checkpointRequestedAt: Date | null; + createdAt: Date; +} + +type Equal = (() => T extends Left ? 1 : 2) extends () => T extends Right ? 1 : 2 ? true : false; +type Assert = T; + +// A future driver gets the same assertions against its own inferred table +// models. This keeps query results identical without sharing table builders. +type _BucketDataMapping = Assert>; +type _BucketParametersMapping = Assert>; +type _CurrentDataMapping = Assert>; +type _SourceTableMapping = Assert>; +type _SyncRulesMapping = Assert>; +type _WriteCheckpointMapping = Assert>; diff --git a/modules/module-drizzle-storage/src/types/types.ts b/modules/module-drizzle-storage/src/types/types.ts new file mode 100644 index 000000000..179278d63 --- /dev/null +++ b/modules/module-drizzle-storage/src/types/types.ts @@ -0,0 +1,37 @@ +import { configFile } from '@powersync/service-types'; +import * as t from 'ts-codec'; + +export const DRIZZLE_SQLITE_STORAGE_TYPE = 'drizzle:sqlite'; + +export const DrizzleSqliteStorageConfig = configFile.BaseStorageConfig.and( + t.object({ + type: t.literal(DRIZZLE_SQLITE_STORAGE_TYPE), + filename: t.string + }) +); + +export type DrizzleSqliteStorageConfig = t.Encoded; +export type DrizzleSqliteStorageConfigDecoded = t.Decoded; +export const DrizzleStorageConfig = DrizzleSqliteStorageConfig; +export type DrizzleStorageConfig = DrizzleSqliteStorageConfig; +export type DrizzleStorageConfigDecoded = DrizzleSqliteStorageConfigDecoded; + +export interface NormalizedDrizzleSqliteStorageConfig { + type: typeof DRIZZLE_SQLITE_STORAGE_TYPE; + filename: string; + max_pool_size: number; +} + +export function isDrizzleStorageConfig(config: configFile.GenericStorageConfig): config is DrizzleSqliteStorageConfig { + return config.type == DRIZZLE_SQLITE_STORAGE_TYPE; +} + +export function normalizeDrizzleSqliteStorageConfig( + config: DrizzleSqliteStorageConfigDecoded +): NormalizedDrizzleSqliteStorageConfig { + return { + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: config.filename, + max_pool_size: config.max_pool_size ?? 10 + }; +} diff --git a/modules/module-drizzle-storage/test/src/__snapshots__/storage.test.ts.snap b/modules/module-drizzle-storage/test/src/__snapshots__/storage.test.ts.snap new file mode 100644 index 000000000..5744533d0 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/__snapshots__/storage.test.ts.snap @@ -0,0 +1,124 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v1 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v1 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v2 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v2 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v3 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Compaction - v3 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`Drizzle SQLite Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; diff --git a/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap new file mode 100644 index 000000000..511e185fd --- /dev/null +++ b/modules/module-drizzle-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -0,0 +1,1190 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`sync - Drizzle SQLite > storage v1 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - Drizzle SQLite > storage v1 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; diff --git a/modules/module-drizzle-storage/test/src/column-types.test.ts b/modules/module-drizzle-storage/test/src/column-types.test.ts new file mode 100644 index 000000000..625cc8359 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/column-types.test.ts @@ -0,0 +1,47 @@ +import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import { describe, expect, it } from 'vitest'; +import { + createSqliteDrizzleRuntime, + DRIZZLE_SQLITE_STORAGE_TYPE, + normalizeDrizzleSqliteStorageConfig, + SQLITE_DRIZZLE_MIGRATIONS_PATH, + sqliteSchema +} from '../../src/index.js'; + +describe('Drizzle SQLite column mappings', () => { + it('round-trips runtime values without bigint precision loss', () => { + const runtime = createSqliteDrizzleRuntime( + normalizeDrizzleSqliteStorageConfig({ + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: ':memory:' + }) + ); + try { + migrate(runtime.db, { migrationsFolder: SQLITE_DRIZZLE_MIGRATIONS_PATH }); + const opId = BigInt(Number.MAX_SAFE_INTEGER) + 100n; + runtime.db + .insert(sqliteSchema.bucketData) + .values({ + id: '1', + groupId: 1, + bucketName: 'bucket', + opId, + op: 'PUT', + checksum: 2n, + sourceTable: 'table', + sourceKey: Buffer.from('key'), + tableName: 'items', + rowId: 'row', + data: '{}', + targetOp: null + }) + .run(); + const row = runtime.db.select().from(sqliteSchema.bucketData).get()!; + expect(row.opId).toBe(opId); + expect(row.checksum).toBe(2n); + expect(row.sourceKey).toEqual(Buffer.from('key')); + } finally { + runtime.close(); + } + }); +}); diff --git a/modules/module-drizzle-storage/test/src/migrations.test.ts b/modules/module-drizzle-storage/test/src/migrations.test.ts new file mode 100644 index 000000000..9944ad090 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/migrations.test.ts @@ -0,0 +1,86 @@ +import { Direction } from '@powersync/lib-services-framework'; +import { rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + DRIZZLE_SQLITE_STORAGE_TYPE, + DrizzleMigrationAgent, + createSqliteDrizzleRuntime, + normalizeDrizzleSqliteStorageConfig +} from '../../src/index.js'; + +describe('Drizzle migrations', () => { + const files: string[] = []; + const filename = () => { + const file = join(tmpdir(), `powersync-drizzle-${process.pid}-${Date.now()}-${files.length}.sqlite`); + files.push(file); + return file; + }; + + afterEach(async () => { + await Promise.all( + files + .splice(0) + .flatMap((file) => [ + rm(file, { force: true }), + rm(`${file}-shm`, { force: true }), + rm(`${file}-wal`, { force: true }) + ]) + ); + }); + + it('creates the storage tables and indexes', async () => { + const file = filename(); + await using agent = new DrizzleMigrationAgent({ type: DRIZZLE_SQLITE_STORAGE_TYPE, filename: file }); + await agent.run({ direction: Direction.Up, migrations: [] }); + + const runtime = createSqliteDrizzleRuntime( + normalizeDrizzleSqliteStorageConfig({ + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: file + }) + ); + try { + const names = runtime.client + .prepare(`SELECT name FROM sqlite_master WHERE type IN ('table', 'index')`) + .all() + .map((row: any) => row.name); + expect(names).toEqual( + expect.arrayContaining([ + 'bucket_data', + 'bucket_parameters', + 'current_data', + 'op_id_sequence', + 'source_tables', + 'sync_rules', + 'write_checkpoints', + 'bucket_data_bucket_op_index', + 'bucket_parameters_lookup_index', + 'write_checkpoints_requested_at_index' + ]) + ); + expect(runtime.client.prepare(`SELECT next_op_id FROM op_id_sequence WHERE id = 1`).get()).toEqual({ + next_op_id: 1n + }); + } finally { + runtime.close(); + } + }); + + it('configures WAL and separate readers for file storage', () => { + const runtime = createSqliteDrizzleRuntime( + normalizeDrizzleSqliteStorageConfig({ + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: filename(), + max_pool_size: 3 + }) + ); + try { + expect(runtime.readers).toHaveLength(2); + expect(runtime.client.pragma('journal_mode', { simple: true })).toBe('wal'); + } finally { + runtime.close(); + } + }); +}); diff --git a/modules/module-drizzle-storage/test/src/setup.ts b/modules/module-drizzle-storage/test/src/setup.ts new file mode 100644 index 000000000..b14ebcec9 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/setup.ts @@ -0,0 +1,11 @@ +import { container } from '@powersync/lib-services-framework'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; +import { beforeAll, beforeEach } from 'vitest'; + +beforeAll(async () => { + container.registerDefaults(); +}); + +beforeEach(async () => { + METRICS_HELPER.resetMetrics(); +}); diff --git a/modules/module-drizzle-storage/test/src/storage-provider.test.ts b/modules/module-drizzle-storage/test/src/storage-provider.test.ts new file mode 100644 index 000000000..0e03335c7 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/storage-provider.test.ts @@ -0,0 +1,34 @@ +import { storage, system } from '@powersync/service-core'; +import { describe, expect, it } from 'vitest'; +import { DRIZZLE_SQLITE_STORAGE_TYPE, DrizzleStorageProvider } from '../../src/index.js'; + +const RESOLVED_CONFIG = { + storage: { type: DRIZZLE_SQLITE_STORAGE_TYPE, filename: ':memory:' }, + slot_name_prefix: 'test_' +} as const; + +function getStorageOptions(serviceMode: system.ServiceContextMode): storage.GetStorageOptions { + return { + resolvedConfig: RESOLVED_CONFIG as unknown as storage.GetStorageOptions['resolvedConfig'], + serviceMode + }; +} + +describe('Drizzle SQLite storage provider', () => { + it('rejects split service runners', async () => { + const provider = new DrizzleStorageProvider(); + await expect(provider.getStorage(getStorageOptions(system.ServiceContextMode.API))).rejects.toThrow( + 'Drizzle SQLite storage only supports the unified service runner' + ); + await expect(provider.getStorage(getStorageOptions(system.ServiceContextMode.SYNC))).rejects.toThrow( + 'Drizzle SQLite storage only supports the unified service runner' + ); + }); + + it('allows the unified service runner', async () => { + const activeStorage = await new DrizzleStorageProvider().getStorage( + getStorageOptions(system.ServiceContextMode.UNIFIED) + ); + await activeStorage.shutDown(); + }); +}); diff --git a/modules/module-drizzle-storage/test/src/storage.test.ts b/modules/module-drizzle-storage/test/src/storage.test.ts new file mode 100644 index 000000000..93ac9c9cb --- /dev/null +++ b/modules/module-drizzle-storage/test/src/storage.test.ts @@ -0,0 +1,21 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { DRIZZLE_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe('Sync Bucket Validation', register.registerBucketValidationTests); + +for (let storageVersion of TEST_STORAGE_VERSIONS) { + describe(`Drizzle SQLite Sync Bucket Storage - Parameters - v${storageVersion}`, () => + register.registerDataStorageParameterTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`Drizzle SQLite Sync Bucket Storage - Data - v${storageVersion}`, () => + register.registerDataStorageDataTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`Drizzle SQLite Sync Bucket Storage - Checkpoints - v${storageVersion}`, () => + register.registerDataStorageCheckpointTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`Drizzle SQLite Sync Bucket Storage - Compaction - v${storageVersion}`, () => { + register.registerCompactTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion }); + register.registerParameterCompactTests({ ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion }); + }); +} diff --git a/modules/module-drizzle-storage/test/src/storage_bench.test.ts b/modules/module-drizzle-storage/test/src/storage_bench.test.ts new file mode 100644 index 000000000..e2f82e334 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/storage_bench.test.ts @@ -0,0 +1,18 @@ +import type { StorageBenchmarkResult } from '@powersync/service-core-tests'; +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { DRIZZLE_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +const results: StorageBenchmarkResult[] = []; +register.registerStorageBenchmarkSummary(results); + +describe.sequential('Drizzle SQLite Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...DRIZZLE_SQLITE_STORAGE_FACTORY, storageVersion }, + { storageName: 'drizzle:sqlite', storageVersion, results } + ); + }); + } +}); diff --git a/modules/module-drizzle-storage/test/src/storage_sync.test.ts b/modules/module-drizzle-storage/test/src/storage_sync.test.ts new file mode 100644 index 000000000..408707c0f --- /dev/null +++ b/modules/module-drizzle-storage/test/src/storage_sync.test.ts @@ -0,0 +1,14 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { DRIZZLE_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe('sync - Drizzle SQLite', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`storage v${storageVersion}`, () => { + register.registerSyncTests(DRIZZLE_SQLITE_STORAGE_FACTORY.factory, { + storageVersion, + tableIdStrings: DRIZZLE_SQLITE_STORAGE_FACTORY.tableIdStrings + }); + }); + } +}); diff --git a/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts b/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts new file mode 100644 index 000000000..c0a1d326a --- /dev/null +++ b/modules/module-drizzle-storage/test/src/sync-rules-storage.test.ts @@ -0,0 +1,173 @@ +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { eq } from 'drizzle-orm'; +import { describe, expect, it } from 'vitest'; +import type { DrizzleBucketStorageFactory } from '../../src/index.js'; +import { DRIZZLE_SQLITE_STORAGE_FACTORY } from './util.js'; + +describe('Drizzle SyncRules storage', () => { + const syncRules = updateSyncRulesFromYaml( + ` +bucket_definitions: + mybucket: + data: [] +`, + { + validate: false + } + ); + + it('stores and resolves managed write checkpoints', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + + const first = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '5/0' } }]) + ).writeCheckpoints.get('user1')!; + const second = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '6/0' } }]) + ).writeCheckpoints.get('user1')!; + + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '4/0' } })).resolves.toBeNull(); + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '5/0' } })).resolves.toBe(first); + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '6/0' } })).resolves.toBe(second); + }); + + it('watches checkpoint and managed write checkpoint changes', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + const abortController = new AbortController(); + + try { + const iterator = bucketStorage + .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) + [Symbol.asyncIterator](); + + const writeCheckpoint = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '5/0' } }]) + ).writeCheckpoints.get('user1')!; + + factory.dialect.db + .update(factory.dialect.tables.syncRules) + .set({ + lastCheckpoint: 0n, + lastCheckpointLsn: '5/0' + }) + .where(eq(factory.dialect.tables.syncRules.id, stream.replicationStreamId)) + .run(); + factory.checkpointWatcher.notify(); + + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { + base: { + checkpoint: 0n, + lsn: '5/0' + }, + writeCheckpoint + } + }); + } finally { + abortController.abort(); + } + }); + + it('resolves custom write checkpoints from the write checkpoint entity', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); + + factory.dialect.db + .insert(factory.dialect.tables.writeCheckpoints) + .values({ + id: 'custom-user1', + syncRulesId: stream.replicationStreamId, + userId: 'user1', + checkpoint: 42n, + heads: null, + checkpointRequestedAt: null, + createdAt: new Date() + }) + .run(); + + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1' })).resolves.toBe(42n); + }); + + it('marks newly resolved source tables as requiring an initial snapshot', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +bucket_definitions: + global: + data: + - SELECT * FROM lists +`, + { + validate: false + } + ) + ); + const bucketStorage = factory.getInstance(stream); + await using writer = await bucketStorage.createWriter({ + defaultSchema: 'public', + zeroLSN: '0/0', + storeCurrentData: true + }); + + const resolved = await writer.resolveTables({ + connection_id: 1, + source: { + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: 123, + schema: 'public', + name: 'lists', + replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] + }, + idGenerator: () => 'lists-table' + }); + + expect(resolved.tables[0]?.snapshotComplete).toBe(false); + + await writer.markTableSnapshotDone(resolved.tables, '0/1'); + const resolvedAgain = await writer.resolveTables({ + connection_id: 1, + source: { + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: 123, + schema: 'public', + name: 'lists', + replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] + } + }); + + expect(resolvedAgain.tables[0]?.snapshotComplete).toBe(true); + }); + + it('returns active and processing streams as replicating streams', async () => { + await using factory = (await DRIZZLE_SQLITE_STORAGE_FACTORY.factory()) as DrizzleBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId, + state: storage.SyncRuleState.PROCESSING + } + ]); + + factory.dialect.db + .update(factory.dialect.tables.syncRules) + .set({ state: storage.SyncRuleState.ACTIVE }) + .where(eq(factory.dialect.tables.syncRules.id, stream.replicationStreamId)) + .run(); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId, + state: storage.SyncRuleState.ACTIVE + } + ]); + }); +}); diff --git a/modules/module-drizzle-storage/test/src/util.ts b/modules/module-drizzle-storage/test/src/util.ts new file mode 100644 index 000000000..5fd6010e1 --- /dev/null +++ b/modules/module-drizzle-storage/test/src/util.ts @@ -0,0 +1,27 @@ +import { storage } from '@powersync/service-core'; +import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import { + createSqliteDrizzleStorageFactory, + DRIZZLE_SQLITE_STORAGE_TYPE, + normalizeDrizzleSqliteStorageConfig, + SQLITE_DRIZZLE_MIGRATIONS_PATH +} from '../../src/index.js'; + +const BASE_CONFIG = { + type: DRIZZLE_SQLITE_STORAGE_TYPE, + filename: ':memory:' +} as const; + +export const DRIZZLE_SQLITE_STORAGE_FACTORY: storage.TestStorageConfig = { + tableIdStrings: true, + factory: async () => { + const factory = createSqliteDrizzleStorageFactory({ + config: normalizeDrizzleSqliteStorageConfig(BASE_CONFIG), + slotNamePrefix: 'test_' + }); + migrate(factory.runtime.db, { migrationsFolder: SQLITE_DRIZZLE_MIGRATIONS_PATH }); + return factory; + } +}; + +export const TEST_STORAGE_VERSIONS = [storage.LEGACY_STORAGE_VERSION]; diff --git a/modules/module-drizzle-storage/test/tsconfig.json b/modules/module-drizzle-storage/test/tsconfig.json new file mode 100644 index 000000000..6538b360a --- /dev/null +++ b/modules/module-drizzle-storage/test/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../../tsconfig.tests.json", + "compilerOptions": { + "declarationDir": "dist/@types", + "tsBuildInfoFile": "dist/.tsbuildinfo", + "lib": ["ES2022", "esnext.disposable"], + "rootDir": "src" + }, + "include": ["src"], + "references": [{ "path": "../" }, { "path": "../../../packages/service-core-tests" }] +} diff --git a/modules/module-drizzle-storage/tsconfig.json b/modules/module-drizzle-storage/tsconfig.json new file mode 100644 index 000000000..9687a076b --- /dev/null +++ b/modules/module-drizzle-storage/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [ + { "path": "../../packages/types" }, + { "path": "../../packages/sync-rules" }, + { "path": "../../packages/service-core" }, + { "path": "../../libs/lib-services" } + ] +} diff --git a/modules/module-drizzle-storage/vitest.config.ts b/modules/module-drizzle-storage/vitest.config.ts new file mode 100644 index 000000000..285c29477 --- /dev/null +++ b/modules/module-drizzle-storage/vitest.config.ts @@ -0,0 +1,7 @@ +import { serviceIntegrationTestConfig } from '../test_config'; + +const config = serviceIntegrationTestConfig(__dirname); +config.test ??= {}; +config.test.testTimeout = 30_000; + +export default config; diff --git a/modules/module-mikroorm-storage/README.md b/modules/module-mikroorm-storage/README.md new file mode 100644 index 000000000..e0e886b6e --- /dev/null +++ b/modules/module-mikroorm-storage/README.md @@ -0,0 +1,169 @@ +# MikroORM Bucket Storage + +Experimental MikroORM-backed bucket storage for the PowerSync service. + +This module currently supports SQLite through the `mikroorm:sqlite` storage type and has an initial MySQL dialect through `mikroorm:mysql`. It is intended to prove out a shared MikroORM storage layer that can support more database drivers while keeping most bucket storage behavior in common code. + +## Status + +This module is experimental. The SQLite implementation is useful for local development, tests, and validating the MikroORM storage architecture. It is not yet intended as the default production storage backend. + +SQLite storage should only be used when running PowerSync in unified mode. Split API and sync runners are rejected because SQLite checkpoint notifications are process-local and cannot notify separate service processes or pods. + +## How It Works + +The module stores PowerSync bucket state in MikroORM entities: + +- `sync_rules` stores deployed sync rule versions and replication stream state. +- `source_tables` stores source table metadata and snapshot progress. +- `bucket_data` stores bucket operations. +- `bucket_parameters` stores materialized parameter lookups. +- `current_data` stores the latest known source-row state used by write batching and compaction. +- `write_checkpoints` stores custom write checkpoint state. +- `instance` stores the PowerSync storage instance id. + +Common entity definitions live in `src/entities/common`. The SQLite and MySQL drivers reuse those definitions directly instead of redeclaring driver-specific entity classes. + +Common storage classes implement most behavior: + +- `MikroOrmBucketStorageFactory` manages sync rule versions and storage instances. +- `MikroOrmSyncRulesStorage` implements the sync-rule-specific storage surface. +- `MikroOrmBucketBatch` owns the writer lifecycle, checkpoints, truncates, and snapshot state. +- `MikroOrmPersistedBatch` persists write chunks in smaller transactions. +- `MikroOrmCompactor` compacts bucket history. +- `MikroOrmStorageDialect` isolates driver-specific streaming and checkpoint notification behavior. + +Migrations are generated and run by MikroORM, but exposed through the standard PowerSync migration surface. Migration execution is guarded by a database-backed lock. For SQLite, that lock table is bootstrapped with raw SQLite before MikroORM migrations run, because migration locking has to work before the normal storage tables exist. + +## Self-Hosted Configuration + +Use `storage.type: mikroorm:sqlite` and provide a SQLite `filename`. + +File-backed SQLite example: + +```yaml +replication: + connections: + - type: postgresql + uri: !env PS_DATA_SOURCE_URI + sslmode: disable + +storage: + type: mikroorm:sqlite + filename: ./powersync-storage.sqlite + +port: 8080 + +sync_rules: + path: sync-rules.yaml + +client_auth: + jwks_uri: !env PS_JWKS_URI + audience: ['powersync'] +``` + +In-memory SQLite example for local tests and throwaway development: + +```yaml +replication: + connections: + - type: postgresql + uri: postgres://postgres:mypassword@localhost:5432/postgres + sslmode: disable + +storage: + type: mikroorm:sqlite + filename: ':memory:' + +port: 8080 + +sync_rules: + path: sync-rules.yaml +``` + +Only use this storage type when starting PowerSync in unified mode. Do not run separate API and sync runners against the same SQLite storage file. + +MySQL example: + +```yaml +replication: + connections: + - type: postgresql + uri: !env PS_DATA_SOURCE_URI + sslmode: disable + +storage: + type: mikroorm:mysql + uri: !env PS_STORAGE_MYSQL_URI + +port: 8080 + +sync_rules: + path: sync-rules.yaml +``` + +The MySQL dialect is newer than SQLite and should be treated as experimental. It currently uses the common in-process checkpoint watcher; database-backed notifications can be added behind `MikroOrmStorageDialect` when needed. + +## SQLite Concurrency + +For file-backed SQLite, the module enables WAL mode and uses MikroORM read replicas to open separate SQLite handles when `storage.max_pool_size` is greater than `1`. This allows SQLite-level read-while-write behavior for API reads during replication writes. + +The current MikroORM SQLite stack uses Kysely's `better-sqlite3` dialect. Those query calls are asynchronous in shape, but they execute on synchronous `better-sqlite3` handles rather than being delegated to worker threads. This means WAL and read replicas improve database handle concurrency, but a long-running SQLite statement can still occupy the Node.js event loop for the process executing it. + +The PowerSync SDK has a Node SQLite dialect that delegates SQLite work to workers. That may be a future workaround if this module needs stronger read concurrency while retaining SQLite storage. + +## Service Registration + +The service image registers this module dynamically under the storage keys `mikroorm:sqlite` and `mikroorm:mysql`. The service package depends on `@powersync/service-module-mikroorm-storage`, and `service/src/util/modules.ts` loads `MikroOrmStorageModule` when the config uses one of these storage types. + +The module also contributes its config type to the generated PowerSync config schema. + +## Development + +Use pnpm through Corepack: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage build +corepack pnpm --filter @powersync/service-module-mikroorm-storage build:tests +corepack pnpm --filter @powersync/service-module-mikroorm-storage test --run +``` + +Focused sync suite: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage test test/src/storage_sync.test.ts --run +``` + +Opt into MySQL storage tests by setting a test database URI: + +```sh +MIKROORM_MYSQL_STORAGE_TEST_URI=mysql://repl_user:good_password@localhost:3306/powersync \ + corepack pnpm --filter @powersync/service-module-mikroorm-storage test test/src/mysql-storage.test.ts --run +``` + +Generate a new SQLite migration from entity changes: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage mikroorm:generate-migration:sqlite +``` + +Generate a MySQL migration from entity changes: + +```sh +MIKRO_ORM_MYSQL_URI=mysql://repl_user:good_password@localhost:3306/powersync \ + corepack pnpm --filter @powersync/service-module-mikroorm-storage mikroorm:generate-migration:mysql +``` + +Generate migrations for all supported dialects: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage mikroorm:generate-migrations +``` + +Generate initial migrations for all supported dialects: + +```sh +corepack pnpm --filter @powersync/service-module-mikroorm-storage mikroorm:generate-initial-migrations +``` + +Review generated migrations before committing them. diff --git a/modules/module-mikroorm-storage/package.json b/modules/module-mikroorm-storage/package.json new file mode 100644 index 000000000..657e5a9cc --- /dev/null +++ b/modules/module-mikroorm-storage/package.json @@ -0,0 +1,59 @@ +{ + "name": "@powersync/service-module-mikroorm-storage", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/index.d.ts", + "version": "0.1.0", + "main": "dist/index.js", + "license": "FSL-1.1-ALv2", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "rm -rf ./dist ./tsconfig.tsbuildinfo && tsc -b", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./dist && tsc -b --clean", + "mikroorm:generate-migration": "pnpm build && mikro-orm migration:create --config dist/mikro-orm.config.js", + "mikroorm:generate-migrations": "pnpm mikroorm:generate-migration:sqlite && pnpm mikroorm:generate-migration:mysql", + "mikroorm:generate-initial-migration": "pnpm build && mikro-orm migration:create --initial --config dist/mikro-orm.config.js", + "mikroorm:generate-initial-migrations": "pnpm mikroorm:generate-initial-migration:sqlite && pnpm mikroorm:generate-initial-migration:mysql", + "mikroorm:generate-initial-migration:mysql": "MIKRO_ORM_STORAGE_DIALECT=mysql pnpm mikroorm:generate-initial-migration", + "mikroorm:generate-initial-migration:sqlite": "MIKRO_ORM_STORAGE_DIALECT=sqlite pnpm mikroorm:generate-initial-migration", + "mikroorm:generate-migration:mysql": "MIKRO_ORM_STORAGE_DIALECT=mysql pnpm mikroorm:generate-migration", + "mikroorm:generate-migration:sqlite": "MIKRO_ORM_STORAGE_DIALECT=sqlite pnpm mikroorm:generate-migration", + "test": "vitest" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "types": "./dist/types/types.d.ts", + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js" + } + }, + "dependencies": { + "@mikro-orm/core": "^7.1.4", + "@mikro-orm/migrations": "^7.1.4", + "@mikro-orm/mysql": "^7.1.4", + "@mikro-orm/sql": "^7.1.4", + "@mikro-orm/sqlite": "^7.1.4", + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-core": "workspace:*", + "@powersync/service-jsonbig": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "ts-codec": "^1.3.0", + "uuid": "catalog:" + }, + "devDependencies": { + "@mikro-orm/cli": "^7.1.4", + "@powersync/service-core-tests": "workspace:*", + "typescript": "catalog:" + } +} diff --git a/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMigrationLockManager.ts b/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMigrationLockManager.ts new file mode 100644 index 000000000..914682e8a --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMigrationLockManager.ts @@ -0,0 +1,79 @@ +import { AbstractMikroOrmMigrationLockManager } from '../../migrations/AbstractMikroOrmMigrationLockManager.js'; + +/** + * MySQL-backed migration lock manager. + * + * The lock table is created with raw SQL because migration locking has to work before MikroORM-created storage tables + * exist. This avoids the classic chicken-and-egg problem where migrations need a lock, but the lock would otherwise + * need a migration-created table. + */ +export class MySqlMigrationLockManager extends AbstractMikroOrmMigrationLockManager { + protected async initLockStore(): Promise { + await this.execute( + ` + CREATE TABLE IF NOT EXISTS powersync_mikroorm_migration_locks ( + name VARCHAR(191) PRIMARY KEY, + lock_id VARCHAR(36) NOT NULL, + expires_at DATETIME(3) NOT NULL, + updated_at DATETIME(3) NOT NULL + ) + `, + [] + ); + } + + protected async tryAcquireLock(options: { + name: string; + lockId: string; + now: Date; + expiresAt: Date; + }): Promise { + const result = await this.execute( + ` + INSERT INTO powersync_mikroorm_migration_locks (name, lock_id, expires_at, updated_at) + VALUES (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + lock_id = IF(expires_at <= ?, VALUES(lock_id), lock_id), + expires_at = IF(expires_at <= ?, VALUES(expires_at), expires_at), + updated_at = IF(expires_at <= ?, VALUES(updated_at), updated_at) + `, + [ + options.name, + options.lockId, + options.expiresAt, + options.now, + options.now, + options.now, + options.now + ] + ); + + return (result.affectedRows ?? 0) > 0; + } + + protected async refreshLock(lockId: string): Promise { + await this.execute( + ` + UPDATE powersync_mikroorm_migration_locks + SET expires_at = ?, updated_at = ? + WHERE name = ? AND lock_id = ? + `, + [new Date(Date.now() + this.timeout), new Date(), this.name, lockId] + ); + } + + protected async releaseLock(lockId: string): Promise { + await this.execute( + ` + DELETE FROM powersync_mikroorm_migration_locks + WHERE name = ? AND lock_id = ? + `, + [this.name, lockId] + ); + } + + private async execute(sql: string, params: unknown[]): Promise<{ affectedRows?: number }> { + const orm = await this.getOrm(); + return orm.em.getConnection().execute(sql, params, 'run'); + } +} diff --git a/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMikroOrmStorageFactory.ts b/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMikroOrmStorageFactory.ts new file mode 100644 index 000000000..707850743 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/mysql/MySqlMikroOrmStorageFactory.ts @@ -0,0 +1,18 @@ +import { MikroORM } from '@mikro-orm/core'; +import { MikroOrmBucketStorageFactory } from '../../storage/MikroOrmBucketStorageFactory.js'; +import { NormalizedMikroOrmMySqlStorageConfig } from '../../types/types.js'; +import { createMySqlMikroOrm } from './mysql-config.js'; +import { mysqlMikroOrmStorageDialect } from './mysql-dialect.js'; + +export async function createMySqlMikroOrmStorageFactory(options: { + config: NormalizedMikroOrmMySqlStorageConfig; + slotNamePrefix: string; + orm?: MikroORM; +}): Promise { + const orm = options.orm ?? (await createMySqlMikroOrm(options.config)); + return new MikroOrmBucketStorageFactory({ + orm, + dialect: mysqlMikroOrmStorageDialect, + slotNamePrefix: options.slotNamePrefix + }); +} diff --git a/modules/module-mikroorm-storage/src/drivers/mysql/mysql-config.ts b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-config.ts new file mode 100644 index 000000000..fc6718e64 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-config.ts @@ -0,0 +1,39 @@ +import { Migrator } from '@mikro-orm/migrations'; +import { defineConfig, MikroORM, MySqlDriver } from '@mikro-orm/mysql'; +import { configureMySqlEntityColumnTypes } from '../../entities/entity-column-types.js'; +import { NormalizedMikroOrmMySqlStorageConfig } from '../../types/types.js'; +import { mysqlMikroOrmStorageDialect } from './mysql-dialect.js'; + +export const MYSQL_MIKRO_ORM_MIGRATIONS_PATH = new URL('../../migrations/mysql', import.meta.url).pathname; + +export function createMySqlMikroOrmOptions(config: NormalizedMikroOrmMySqlStorageConfig) { + configureMySqlEntityColumnTypes(); + + return defineConfig({ + driver: MySqlDriver, + clientUrl: config.uri, + entities: mysqlMikroOrmStorageDialect.entityClasses, + pool: { + min: 0, + max: config.max_pool_size + }, + extensions: [Migrator], + migrations: { + path: MYSQL_MIKRO_ORM_MIGRATIONS_PATH, + pathTs: MYSQL_MIKRO_ORM_MIGRATIONS_PATH, + glob: '!(*.d).{js,ts}', + emit: 'ts', + snapshot: false, + dropTables: false, + // MySQL DDL performs implicit commits, so wrapping generated migrations in + // MikroORM transactions/savepoints can leave the driver trying to roll + // back a savepoint that MySQL has already discarded. + transactional: false, + allOrNothing: false + } + }); +} + +export async function createMySqlMikroOrm(config: NormalizedMikroOrmMySqlStorageConfig): Promise { + return MikroORM.init(createMySqlMikroOrmOptions(config)); +} diff --git a/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts new file mode 100644 index 000000000..587b7551c --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/mysql/mysql-dialect.ts @@ -0,0 +1,67 @@ +import { + BucketData, + BucketDataSchema, + BucketParameters, + BucketParametersSchema, + CurrentData, + CurrentDataSchema, + Instance, + InstanceSchema, + SourceTable, + SourceTableSchema, + SyncRules, + SyncRulesSchema, + WriteCheckpoint, + WriteCheckpointSchema +} from '../../entities/entities-index.js'; +import { InProcessMikroOrmCheckpointWatcher, MikroOrmStorageDialect } from '../../storage/MikroOrmStorageDialect.js'; +import { MIKRO_ORM_MYSQL_STORAGE_TYPE } from '../../types/types.js'; +import { streamBucketDataRowsFromSql } from '../sql/bucket-data-read.js'; + +export const mysqlMikroOrmStorageDialect: MikroOrmStorageDialect = { + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + entityClasses: [ + BucketDataSchema, + BucketParametersSchema, + CurrentDataSchema, + InstanceSchema, + SourceTableSchema, + SyncRulesSchema, + WriteCheckpointSchema + ], + bucketDataEntity: BucketData, + bucketParametersEntity: BucketParameters, + currentDataEntity: CurrentData, + instanceEntity: Instance, + sourceTableEntity: SourceTable, + syncRulesEntity: SyncRules, + writeCheckpointEntity: WriteCheckpoint, + async *streamBucketDataRows(options) { + if (options.dataBuckets.length == 0) { + return; + } + + yield* streamBucketDataRowsFromSql(options, (queryOptions) => { + const requestedRows = queryOptions.dataBuckets + .map((_, index) => `${index == 0 ? 'SELECT' : 'UNION ALL SELECT'} ? AS bucket_name, ? AS start_op_id`) + .join(' '); + const params = queryOptions.dataBuckets.flatMap((request) => [request.bucket, request.start]); + + return { + sql: ` + SELECT bucket_data.* + FROM (${requestedRows}) AS requested + JOIN bucket_data FORCE INDEX (bucket_data_bucket_op_index) + ON bucket_data.group_id = ? + AND bucket_data.bucket_name = requested.bucket_name + AND bucket_data.op_id > requested.start_op_id + AND bucket_data.op_id <= ? + ORDER BY bucket_data.bucket_name ASC, bucket_data.op_id ASC + LIMIT ? + `, + params: [...params, queryOptions.groupId, queryOptions.checkpoint, queryOptions.limit] + }; + }); + }, + createCheckpointWatcher: () => new InProcessMikroOrmCheckpointWatcher() +}; diff --git a/modules/module-mikroorm-storage/src/drivers/sql/bucket-data-read.ts b/modules/module-mikroorm-storage/src/drivers/sql/bucket-data-read.ts new file mode 100644 index 000000000..4df2003c9 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sql/bucket-data-read.ts @@ -0,0 +1,71 @@ +import type { SqlEntityManager } from '@mikro-orm/sql'; +import type { BucketData } from '../../entities/entities-index.js'; +import type { MikroOrmBucketDataStreamOptions } from '../../storage/MikroOrmStorageDialect.js'; + +const REQUEST_BUCKET_CHUNK_SIZE = 400; + +export interface SqlBucketDataReadQueryOptions extends MikroOrmBucketDataStreamOptions { + readonly limit: number; +} + +export interface SqlBucketDataReadQuery { + readonly sql: string; + readonly params: unknown[]; +} + +export async function* streamBucketDataRowsFromSql( + options: MikroOrmBucketDataStreamOptions, + buildQuery: (options: SqlBucketDataReadQueryOptions) => SqlBucketDataReadQuery +): AsyncIterable { + const sqlEntityManager = options.em as SqlEntityManager; + const sortedBuckets = [...options.dataBuckets].sort((a, b) => a.bucket.localeCompare(b.bucket)); + let remainingLimit = options.limit; + + for (let offset = 0; offset < sortedBuckets.length && remainingLimit > 0; offset += REQUEST_BUCKET_CHUNK_SIZE) { + const dataBuckets = sortedBuckets.slice(offset, offset + REQUEST_BUCKET_CHUNK_SIZE); + const query = buildQuery({ + ...options, + dataBuckets, + limit: remainingLimit + }); + const rows = await sqlEntityManager.getConnection().execute(query.sql, query.params, 'all'); + + for (const row of rows) { + yield rawBucketDataRow(row); + } + + remainingLimit -= rows.length; + } +} + +interface RawBucketDataRow { + id: string; + group_id: number; + bucket_name: string; + op_id: bigint | number | string; + op: string; + source_table: string | null; + source_key: Uint8Array | null; + table_name: string | null; + row_id: string | null; + checksum: bigint | number | string; + data: string | null; + target_op: bigint | number | string | null; +} + +function rawBucketDataRow(row: RawBucketDataRow): BucketData { + return { + id: row.id, + groupId: row.group_id, + bucketName: row.bucket_name, + opId: BigInt(row.op_id), + op: row.op, + sourceTable: row.source_table, + sourceKey: row.source_key, + tableName: row.table_name, + rowId: row.row_id, + checksum: BigInt(row.checksum), + data: row.data, + targetOp: row.target_op == null ? null : BigInt(row.target_op) + }; +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts new file mode 100644 index 000000000..d20030940 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMigrationLockManager.ts @@ -0,0 +1,89 @@ +import { AbstractMikroOrmMigrationLockManager } from '../../migrations/AbstractMikroOrmMigrationLockManager.js'; + +/** + * SQLite implementation of the migration lock. + * + * This intentionally uses raw SQLite statements instead of MikroORM entities. Migration locking has a classic + * chicken-and-egg problem: the service must take a DB-backed lock before it asks MikroORM to run the migrations + * that normally create and evolve application tables. The lock table therefore has to bootstrap itself with + * `CREATE TABLE IF NOT EXISTS`, outside the generated MikroORM migration model. + */ +export class SqliteMigrationLockManager extends AbstractMikroOrmMigrationLockManager { + protected async initLockStore(): Promise { + await this.execute( + ` + CREATE TABLE IF NOT EXISTS powersync_migration_locks ( + name TEXT PRIMARY KEY, + lock_id TEXT, + expires_at INTEGER NOT NULL + ) + `, + [] + ); + + await this.execute( + ` + INSERT OR IGNORE INTO powersync_migration_locks (name, lock_id, expires_at) + VALUES (?, NULL, 0) + `, + [this.name] + ); + } + + protected async tryAcquireLock(options: { + name: string; + lockId: string; + now: Date; + expiresAt: Date; + }): Promise { + const result = await this.execute( + ` + UPDATE powersync_migration_locks + SET lock_id = ?, expires_at = ? + WHERE name = ? + AND (lock_id IS NULL OR expires_at <= ?) + `, + [options.lockId, options.expiresAt.getTime(), options.name, options.now.getTime()] + ); + + return result.affectedRows == 1; + } + + protected async refreshLock(lockId: string): Promise { + const result = await this.execute( + ` + UPDATE powersync_migration_locks + SET expires_at = ? + WHERE name = ? AND lock_id = ? + `, + [new Date(Date.now() + this.timeout).getTime(), this.name, lockId] + ); + + if (result.affectedRows != 1) { + throw new Error('Lock not found, could not refresh'); + } + } + + protected async releaseLock(lockId: string): Promise { + const result = await this.execute( + ` + UPDATE powersync_migration_locks + SET lock_id = NULL, expires_at = 0 + WHERE name = ? AND lock_id = ? + `, + [this.name, lockId] + ); + + if (result.affectedRows != 1) { + throw new Error('Lock not found, could not release'); + } + } + + /** + * Execute raw SQLite against MikroORM's connection so the lock can exist before entity-backed schema exists. + */ + private async execute(sql: string, params: unknown[]): Promise<{ affectedRows?: number }> { + const orm = await this.getOrm(); + return orm.em.getConnection().execute(sql, params, 'run'); + } +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMikroOrmStorageFactory.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMikroOrmStorageFactory.ts new file mode 100644 index 000000000..d7a845e60 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/SqliteMikroOrmStorageFactory.ts @@ -0,0 +1,18 @@ +import { MikroORM } from '@mikro-orm/core'; +import { MikroOrmBucketStorageFactory } from '../../storage/MikroOrmBucketStorageFactory.js'; +import { NormalizedMikroOrmSqliteStorageConfig } from '../../types/types.js'; +import { createSqliteMikroOrm } from './sqlite-config.js'; +import { sqliteMikroOrmStorageDialect } from './sqlite-dialect.js'; + +export async function createSqliteMikroOrmStorageFactory(options: { + config: NormalizedMikroOrmSqliteStorageConfig; + slotNamePrefix: string; + orm?: MikroORM; +}): Promise { + const orm = options.orm ?? (await createSqliteMikroOrm(options.config)); + return new MikroOrmBucketStorageFactory({ + orm, + dialect: sqliteMikroOrmStorageDialect, + slotNamePrefix: options.slotNamePrefix + }); +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts new file mode 100644 index 000000000..ef70f6c32 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-config.ts @@ -0,0 +1,59 @@ +import { Migrator } from '@mikro-orm/migrations'; +import { defineConfig, MikroORM, SqliteDriver } from '@mikro-orm/sqlite'; +import { configureDefaultEntityColumnTypes } from '../../entities/entity-column-types.js'; +import { NormalizedMikroOrmSqliteStorageConfig } from '../../types/types.js'; +import { sqliteMikroOrmStorageDialect } from './sqlite-dialect.js'; + +export const SQLITE_MIKRO_ORM_MIGRATIONS_PATH = new URL('../../migrations/sqlite', import.meta.url).pathname; + +export function createSqliteMikroOrmOptions(config: NormalizedMikroOrmSqliteStorageConfig) { + configureDefaultEntityColumnTypes(); + + const fileBacked = config.filename != ':memory:'; + const readReplicaCount = fileBacked ? Math.max(0, config.max_pool_size - 1) : 0; + + return defineConfig({ + driver: SqliteDriver, + dbName: config.filename, + entities: sqliteMikroOrmStorageDialect.entityClasses, + onCreateConnection: (connection) => + configureSqliteConnection(connection, { + enableWal: fileBacked + }), + replicas: Array.from({ length: readReplicaCount }, (_, index) => ({ + name: `reader-${index + 1}` + })), + extensions: [Migrator], + migrations: { + path: SQLITE_MIKRO_ORM_MIGRATIONS_PATH, + pathTs: SQLITE_MIKRO_ORM_MIGRATIONS_PATH, + glob: '!(*.d).{js,ts}', + emit: 'ts', + snapshot: false, + dropTables: false + } + }); +} + +export async function createSqliteMikroOrm(config: NormalizedMikroOrmSqliteStorageConfig): Promise { + return MikroORM.init(createSqliteMikroOrmOptions(config)); +} + +interface SqliteKyselyConnection { + executeQuery(query: { sql: string; parameters: unknown[] }): Promise; +} + +async function configureSqliteConnection( + connection: unknown, + options: { + enableWal: boolean; + } +): Promise { + const sqliteConnection = connection as SqliteKyselyConnection; + + if (options.enableWal) { + await sqliteConnection.executeQuery({ sql: 'PRAGMA journal_mode = WAL', parameters: [] }); + } + + await sqliteConnection.executeQuery({ sql: 'PRAGMA busy_timeout = 5000', parameters: [] }); +} diff --git a/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts new file mode 100644 index 000000000..31bc76363 --- /dev/null +++ b/modules/module-mikroorm-storage/src/drivers/sqlite/sqlite-dialect.ts @@ -0,0 +1,93 @@ +import type { SqlEntityManager } from '@mikro-orm/sql'; +import { + BucketData, + BucketDataSchema, + BucketParameters, + BucketParametersSchema, + CurrentData, + CurrentDataSchema, + Instance, + InstanceSchema, + SourceTable, + SourceTableSchema, + SyncRules, + SyncRulesSchema, + WriteCheckpoint, + WriteCheckpointSchema +} from '../../entities/entities-index.js'; +import { InProcessMikroOrmCheckpointWatcher, MikroOrmStorageDialect } from '../../storage/MikroOrmStorageDialect.js'; +import { MIKRO_ORM_SQLITE_STORAGE_TYPE } from '../../types/types.js'; + +export const sqliteMikroOrmStorageDialect: MikroOrmStorageDialect = { + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + entityClasses: [ + BucketDataSchema, + BucketParametersSchema, + CurrentDataSchema, + InstanceSchema, + SourceTableSchema, + SyncRulesSchema, + WriteCheckpointSchema + ], + bucketDataEntity: BucketData, + bucketParametersEntity: BucketParameters, + currentDataEntity: CurrentData, + instanceEntity: Instance, + sourceTableEntity: SourceTable, + syncRulesEntity: SyncRules, + writeCheckpointEntity: WriteCheckpoint, + async *streamBucketDataRows(options) { + if (options.dataBuckets.length == 0) { + return; + } + + // SQLite reads can complete synchronously enough that tight polling loops + // starve replication/checkpoint work in the same process. Yield once before + // the query so single-process unified mode remains cooperative. + await new Promise((resolve) => setImmediate(resolve)); + + const sqlEntityManager = options.em as SqlEntityManager; + const sortedBuckets = [...options.dataBuckets].sort((a, b) => a.bucket.localeCompare(b.bucket)); + let remainingLimit = options.limit; + + for (const request of sortedBuckets) { + if (remainingLimit <= 0) { + break; + } + + // Query each bucket as its own indexed range scan. In SQLite this was faster than joining a VALUES table for + // many buckets because it avoids cross-bucket planning and temporary sorting work. + const queryBuilder = sqlEntityManager + .createQueryBuilder(BucketData, 'bucket_data') + .select('*') + .where({ + groupId: options.groupId, + bucketName: request.bucket, + opId: { $gt: request.start, $lte: options.checkpoint } + }) + .orderBy({ opId: 'ASC' }) + .limit(remainingLimit); + + for await (const row of streamQueryBuilder(queryBuilder.stream())) { + yield row; + remainingLimit--; + } + } + }, + createCheckpointWatcher: () => new InProcessMikroOrmCheckpointWatcher() +}; + +async function* streamQueryBuilder(stream: AsyncIterableIterator): AsyncIterable { + const iterator = stream[Symbol.asyncIterator](); + try { + while (true) { + const result = await iterator.next(); + if (result.done) { + return; + } + yield result.value; + } + } finally { + await iterator.return?.(); + } +} diff --git a/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts b/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts new file mode 100644 index 000000000..bbf7eec71 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/bucket-data.schema.ts @@ -0,0 +1,33 @@ +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; + +export const BucketDataSchema = defineEntity({ + name: 'BucketData', + tableName: 'bucket_data', + indexes: [ + { + name: 'bucket_data_bucket_op_index', + properties: ['groupId', 'bucketName', 'opId'] + }, + { + name: 'bucket_data_source_index', + properties: ['groupId', 'sourceTable', 'sourceKey'] + } + ], + properties: { + id: p.string().primary(), + groupId: p.integer().fieldName('group_id'), + bucketName: p.string().fieldName('bucket_name'), + opId: p.bigint('bigint'), + op: p.string(), + sourceTable: p.string().fieldName('source_table').nullable(), + sourceKey: p.blob().nullable(), + tableName: p.string().fieldName('table_name').nullable(), + rowId: p.string().fieldName('row_id').nullable(), + checksum: p.bigint('bigint'), + data: p.text().nullable(), + targetOp: p.bigint('bigint').nullable() + } +}); + +export const BucketData = BucketDataSchema.class; +export type BucketData = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts b/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts new file mode 100644 index 000000000..3a0b6946b --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/bucket-parameters.schema.ts @@ -0,0 +1,27 @@ +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; + +export const BucketParametersSchema = defineEntity({ + name: 'BucketParameters', + tableName: 'bucket_parameters', + indexes: [ + { + name: 'bucket_parameters_lookup_index', + properties: ['groupId', 'lookup', 'id'] + }, + { + name: 'bucket_parameters_source_index', + properties: ['groupId', 'sourceTable', 'sourceKey'] + } + ], + properties: { + id: p.bigint('bigint').primary().autoincrement(false), + groupId: p.integer().fieldName('group_id'), + sourceTable: p.string().fieldName('source_table'), + sourceKey: p.blob(), + lookup: p.blob(), + bucketParameters: p.json() + } +}); + +export const BucketParameters = BucketParametersSchema.class; +export type BucketParameters = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts b/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts new file mode 100644 index 000000000..654888e1f --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/current-data.schema.ts @@ -0,0 +1,29 @@ +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; + +export const CurrentDataSchema = defineEntity({ + name: 'CurrentData', + tableName: 'current_data', + indexes: [ + { + name: 'current_data_source_index', + properties: ['groupId', 'sourceTable', 'sourceKey'] + }, + { + name: 'current_data_pending_delete_index', + properties: ['groupId', 'pendingDelete'] + } + ], + properties: { + id: p.string().primary(), + groupId: p.integer().fieldName('group_id'), + sourceTable: p.string().fieldName('source_table'), + sourceKey: p.blob(), + buckets: p.json(), + lookups: p.json(), + data: p.blob(), + pendingDelete: p.bigint('bigint').nullable() + } +}); + +export const CurrentData = CurrentDataSchema.class; +export type CurrentData = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts b/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts new file mode 100644 index 000000000..87c046809 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/instance.schema.ts @@ -0,0 +1,12 @@ +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; + +export const InstanceSchema = defineEntity({ + name: 'Instance', + tableName: 'instance', + properties: { + id: p.string().primary() + } +}); + +export const Instance = InstanceSchema.class; +export type Instance = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts b/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts new file mode 100644 index 000000000..4289a9077 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/source-table.schema.ts @@ -0,0 +1,28 @@ +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; + +export const SourceTableSchema = defineEntity({ + name: 'SourceTable', + tableName: 'source_tables', + indexes: [ + { + name: 'source_table_lookup', + properties: ['groupId', 'tableName'] + } + ], + properties: { + id: p.string().primary(), + groupId: p.integer().fieldName('group_id'), + connectionId: p.integer().fieldName('connection_id'), + relationId: p.json().nullable(), + schemaName: p.string().fieldName('schema_name'), + tableName: p.string().fieldName('table_name'), + replicaIdColumns: p.json().nullable(), + snapshotDone: p.boolean().fieldName('snapshot_done').default(true), + snapshotTotalEstimatedCount: p.bigint('bigint').nullable(), + snapshotReplicatedCount: p.bigint('bigint').nullable(), + snapshotLastKey: p.blob().nullable() + } +}); + +export const SourceTable = SourceTableSchema.class; +export type SourceTable = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts b/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts new file mode 100644 index 000000000..91f770718 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/sync-rules.schema.ts @@ -0,0 +1,28 @@ +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; +import type { storage } from '@powersync/service-core'; + +export const SyncRulesSchema = defineEntity({ + name: 'SyncRules', + tableName: 'sync_rules', + properties: { + id: p.integer().primary().autoincrement(), + state: p.string().$type(), + snapshotDone: p.boolean().fieldName('snapshot_done').default(false), + snapshotLsn: p.string().fieldName('snapshot_lsn').nullable(), + lastCheckpoint: p.bigint('bigint').nullable(), + lastCheckpointLsn: p.string().fieldName('last_checkpoint_lsn').strictNullable(), + noCheckpointBefore: p.string().fieldName('no_checkpoint_before').nullable(), + slotName: p.string().fieldName('slot_name'), + lastCheckpointTs: p.datetime().nullable(), + lastKeepaliveTs: p.datetime().nullable(), + lastFatalError: p.string().fieldName('last_fatal_error').strictNullable(), + lastFatalErrorTs: p.datetime().strictNullable(), + keepaliveOp: p.bigint('bigint').nullable(), + storageVersion: p.integer().fieldName('storage_version').nullable(), + content: p.text(), + syncPlan: p.json().strictNullable() + } +}); + +export const SyncRules = SyncRulesSchema.class; +export type SyncRules = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts b/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts new file mode 100644 index 000000000..fa8ba474b --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/common/write-checkpoint.schema.ts @@ -0,0 +1,28 @@ +import { defineEntity, p, type InferEntity } from '@mikro-orm/core'; + +export const WriteCheckpointSchema = defineEntity({ + name: 'WriteCheckpoint', + tableName: 'write_checkpoints', + indexes: [ + { + name: 'write_checkpoints_user_checkpoint_index', + properties: ['userId', 'syncRulesId', 'checkpoint'] + }, + { + name: 'write_checkpoints_requested_at_index', + properties: ['checkpointRequestedAt'] + } + ], + properties: { + id: p.string().primary(), + syncRulesId: p.integer().fieldName('sync_rules_id').nullable(), + userId: p.string().fieldName('user_id'), + checkpoint: p.bigint('bigint'), + heads: p.json().nullable(), + checkpointRequestedAt: p.datetime().fieldName('checkpoint_requested_at').nullable(), + createdAt: p.datetime() + } +}); + +export const WriteCheckpoint = WriteCheckpointSchema.class; +export type WriteCheckpoint = InferEntity; diff --git a/modules/module-mikroorm-storage/src/entities/entities-index.ts b/modules/module-mikroorm-storage/src/entities/entities-index.ts new file mode 100644 index 000000000..13728f1cb --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/entities-index.ts @@ -0,0 +1,7 @@ +export * from './common/bucket-data.schema.js'; +export * from './common/bucket-parameters.schema.js'; +export * from './common/current-data.schema.js'; +export * from './common/instance.schema.js'; +export * from './common/source-table.schema.js'; +export * from './common/sync-rules.schema.js'; +export * from './common/write-checkpoint.schema.js'; diff --git a/modules/module-mikroorm-storage/src/entities/entity-column-types.ts b/modules/module-mikroorm-storage/src/entities/entity-column-types.ts new file mode 100644 index 000000000..46bce46c8 --- /dev/null +++ b/modules/module-mikroorm-storage/src/entities/entity-column-types.ts @@ -0,0 +1,43 @@ +import { + BucketDataSchema, + BucketParametersSchema, + CurrentDataSchema, + SyncRulesSchema +} from './entities-index.js'; + +const MYSQL_INDEXED_BINARY_COLUMN_TYPE = 'varbinary(1024)'; +const MYSQL_LONG_TEXT_COLUMN_TYPE = 'longtext'; +const MYSQL_LONG_BINARY_COLUMN_TYPE = 'longblob'; + +export function configureDefaultEntityColumnTypes(): void { + setColumnType(BucketDataSchema, 'sourceKey', undefined); + setColumnType(BucketDataSchema, 'data', undefined); + setColumnType(BucketParametersSchema, 'sourceKey', undefined); + setColumnType(BucketParametersSchema, 'lookup', undefined); + setColumnType(CurrentDataSchema, 'sourceKey', undefined); + setColumnType(CurrentDataSchema, 'data', undefined); + setColumnType(SyncRulesSchema, 'content', undefined); +} + +export function configureMySqlEntityColumnTypes(): void { + setColumnType(BucketDataSchema, 'sourceKey', MYSQL_INDEXED_BINARY_COLUMN_TYPE); + setColumnType(BucketDataSchema, 'data', MYSQL_LONG_TEXT_COLUMN_TYPE); + setColumnType(BucketParametersSchema, 'sourceKey', MYSQL_INDEXED_BINARY_COLUMN_TYPE); + setColumnType(BucketParametersSchema, 'lookup', MYSQL_INDEXED_BINARY_COLUMN_TYPE); + setColumnType(CurrentDataSchema, 'sourceKey', MYSQL_INDEXED_BINARY_COLUMN_TYPE); + setColumnType(CurrentDataSchema, 'data', MYSQL_LONG_BINARY_COLUMN_TYPE); + setColumnType(SyncRulesSchema, 'content', MYSQL_LONG_TEXT_COLUMN_TYPE); +} + +function setColumnType(schema: { properties: Record }, propertyName: string, columnType: string | undefined) { + const property = schema.properties[propertyName] as { columnType?: string } | undefined; + if (property == null) { + throw new Error(`Cannot configure missing MikroORM entity property ${propertyName}`); + } + + if (columnType == null) { + delete property.columnType; + } else { + property.columnType = columnType; + } +} diff --git a/modules/module-mikroorm-storage/src/index.ts b/modules/module-mikroorm-storage/src/index.ts new file mode 100644 index 000000000..30dae6129 --- /dev/null +++ b/modules/module-mikroorm-storage/src/index.ts @@ -0,0 +1,13 @@ +export * from './drivers/mysql/mysql-config.js'; +export * from './drivers/mysql/mysql-dialect.js'; +export * from './drivers/mysql/MySqlMikroOrmStorageFactory.js'; +export * from './drivers/sqlite/sqlite-config.js'; +export * from './drivers/sqlite/sqlite-dialect.js'; +export * from './drivers/sqlite/SqliteMikroOrmStorageFactory.js'; +export * from './entities/entities-index.js'; +export * as entities from './entities/entities-index.js'; +export * from './migrations/MikroOrmMigrationAgent.js'; +export * from './module/MikroOrmStorageModule.js'; +export * from './storage/storage-index.js'; +export * as storage from './storage/storage-index.js'; +export * from './types/types.js'; diff --git a/modules/module-mikroorm-storage/src/migrations/AbstractMikroOrmMigrationLockManager.ts b/modules/module-mikroorm-storage/src/migrations/AbstractMikroOrmMigrationLockManager.ts new file mode 100644 index 000000000..89a76b7d2 --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/AbstractMikroOrmMigrationLockManager.ts @@ -0,0 +1,101 @@ +import { MikroORM } from '@mikro-orm/core'; +import { locks } from '@powersync/lib-services-framework'; +import * as uuid from 'uuid'; + +const DEFAULT_LOCK_TIMEOUT = 60_000; + +/** + * Construction parameters for a database-backed MikroORM migration lock manager. + * + * The ORM instance is supplied so dialect implementations can reuse MikroORM's configured connection while still + * issuing the small amount of raw SQL required to bootstrap the lock table before migrations have run. + */ +export interface MikroOrmMigrationLockManagerParams extends locks.LockManagerParams { + orm: MikroORM | Promise; +} + +/** + * Base lock manager for MikroORM-backed storage migrations. + * + * The common layer owns lock lifecycle behavior and expiry handling. Dialects provide the actual persistence + * operations because lock bootstrap is database-specific and must work before the normal MikroORM migration schema + * exists. + */ +export abstract class AbstractMikroOrmMigrationLockManager extends locks.AbstractLockManager { + private ormInstance: MikroORM | undefined; + + constructor(protected params: MikroOrmMigrationLockManagerParams) { + super(params); + } + + protected get timeout() { + return this.params.timeout ?? DEFAULT_LOCK_TIMEOUT; + } + + protected get name() { + return this.params.name; + } + + protected async getOrm(): Promise { + this.ormInstance ??= await this.params.orm; + return this.ormInstance; + } + + async init(): Promise { + await this.initLockStore(); + } + + protected async acquireHandle(): Promise { + const lockId = await this.acquireLockId(); + if (lockId == null) { + return null; + } + + return { + refresh: () => this.refreshLock(lockId), + release: () => this.releaseLock(lockId) + }; + } + + protected async acquireLockId(): Promise { + const lockId = uuid.v4(); + const now = new Date(); + const expiresAt = new Date(now.getTime() + this.timeout); + + return (await this.tryAcquireLock({ + name: this.name, + lockId, + now, + expiresAt + })) + ? lockId + : null; + } + + /** + * Create or prepare the dialect-specific lock storage. + * + * Implementations must be safe to call before any generated MikroORM migration has run. + */ + protected abstract initLockStore(): Promise; + + /** + * Try to acquire the named lock if it is free or expired. + */ + protected abstract tryAcquireLock(options: { + name: string; + lockId: string; + now: Date; + expiresAt: Date; + }): Promise; + + /** + * Extend ownership of the currently-held lock. + */ + protected abstract refreshLock(lockId: string): Promise; + + /** + * Release ownership of the currently-held lock. + */ + protected abstract releaseLock(lockId: string): Promise; +} diff --git a/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts b/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts new file mode 100644 index 000000000..147879c15 --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/MikroOrmMigrationAgent.ts @@ -0,0 +1,127 @@ +import { MikroORM } from '@mikro-orm/core'; +import * as framework from '@powersync/lib-services-framework'; +import { migrations } from '@powersync/service-core'; +import { createMySqlMikroOrm } from '../drivers/mysql/mysql-config.js'; +import { MySqlMigrationLockManager } from '../drivers/mysql/MySqlMigrationLockManager.js'; +import { createSqliteMikroOrm } from '../drivers/sqlite/sqlite-config.js'; +import { SqliteMigrationLockManager } from '../drivers/sqlite/SqliteMigrationLockManager.js'; +import { + MIKRO_ORM_MYSQL_STORAGE_TYPE, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + MikroOrmStorageConfigDecoded, + normalizeMikroOrmMySqlStorageConfig, + normalizeMikroOrmSqliteStorageConfig +} from '../types/types.js'; +import { NoOpMigrationStore } from './NoOpMigrationStore.js'; + +/** + * Service migration agent that delegates schema changes to MikroORM migrations. + * + * PowerSync still owns orchestration and locking, but MikroORM owns migration discovery and migration state. The + * lock is acquired first to avoid multiple service instances running the same MikroORM migration concurrently. + */ +export class MikroOrmMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent { + store: framework.MigrationStore; + locks: framework.LockManager; + + private readonly ormPromise: Promise; + + constructor(config: MikroOrmStorageConfigDecoded) { + super(); + const runtime = createMigrationRuntime(config); + this.ormPromise = runtime.ormPromise; + this.store = new NoOpMigrationStore(); + this.locks = runtime.lockManager; + } + + getInternalScriptsDir(): string { + return new URL('./scripts', import.meta.url).pathname; + } + + async run(params: framework.RunMigrationParams): Promise { + await this.locks.init?.(); + + const logger = params.logger ?? framework.logger; + logger.debug('Acquiring lock for MikroORM migrations'); + const lockHandle = await this.locks.acquire({ + max_wait_ms: params.maxLockWaitMs ?? framework.DEFAULT_MAX_LOCK_WAIT_MS + }); + + if (lockHandle == null) { + throw new Error('Could not acquire MikroORM migration lock'); + } + + let isReleased = false; + const releaseLock = async () => { + if (isReleased) { + return; + } + await lockHandle.release(); + isReleased = true; + }; + + process.addListener('beforeExit', releaseLock); + + try { + if (params.migrations.length > 0) { + logger.warn('Ignoring PowerSync migration list for MikroORM storage; MikroORM migrator owns migration state.'); + } + + const orm = await this.ormPromise; + + logger.info(`Running MikroORM migrations ${params.direction}`); + const migrator = orm.migrator; + if (migrator != null) { + if (params.direction == framework.Direction.Up) { + await migrator.up(); + } else { + await migrator.down(); + } + } else if (params.direction == framework.Direction.Up) { + await orm.schema.update(); + } + } finally { + logger.debug('Releasing MikroORM migration lock'); + await releaseLock(); + process.removeListener('beforeExit', releaseLock); + logger.debug('Done with MikroORM migrations'); + } + } + + async loadInternalMigrations(): Promise[]> { + return []; + } + + async [Symbol.asyncDispose](): Promise { + const orm = await this.ormPromise; + await orm.close(true); + } +} + +function createMigrationRuntime(config: MikroOrmStorageConfigDecoded): { + ormPromise: Promise; + lockManager: framework.LockManager; +} { + switch (config.type) { + case MIKRO_ORM_SQLITE_STORAGE_TYPE: { + const ormPromise = createSqliteMikroOrm(normalizeMikroOrmSqliteStorageConfig(config)); + return { + ormPromise, + lockManager: new SqliteMigrationLockManager({ + name: 'mikroorm-migrations', + orm: ormPromise + }) + }; + } + case MIKRO_ORM_MYSQL_STORAGE_TYPE: { + const ormPromise = createMySqlMikroOrm(normalizeMikroOrmMySqlStorageConfig(config)); + return { + ormPromise, + lockManager: new MySqlMigrationLockManager({ + name: 'mikroorm-migrations', + orm: ormPromise + }) + }; + } + } +} diff --git a/modules/module-mikroorm-storage/src/migrations/NoOpMigrationStore.ts b/modules/module-mikroorm-storage/src/migrations/NoOpMigrationStore.ts new file mode 100644 index 000000000..2f3c5b08d --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/NoOpMigrationStore.ts @@ -0,0 +1,15 @@ +import { migrations } from '@powersync/lib-services-framework'; + +/** + * Service migrations only trigger MikroORM's migrator for this module. + * MikroORM owns the actual migration state in its own migration storage. + */ +export class NoOpMigrationStore implements migrations.MigrationStore { + async load(): Promise { + return undefined; + } + + async save(_state: migrations.MigrationState): Promise {} + + async clear(): Promise {} +} diff --git a/modules/module-mikroorm-storage/src/migrations/mysql/Migration20260612150058.ts b/modules/module-mikroorm-storage/src/migrations/mysql/Migration20260612150058.ts new file mode 100644 index 000000000..a1ada3de9 --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/mysql/Migration20260612150058.ts @@ -0,0 +1,29 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20260612150058 extends Migration { + + override up(): void | Promise { + this.addSql(`create table \`bucket_data\` (\`id\` varchar(255) not null, \`group_id\` int not null, \`bucket_name\` varchar(255) not null, \`op_id\` bigint not null, \`op\` varchar(255) not null, \`source_table\` varchar(255) null, \`source_key\` varbinary(1024) null, \`table_name\` varchar(255) null, \`row_id\` varchar(255) null, \`checksum\` bigint not null, \`data\` longtext null, \`target_op\` bigint null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`bucket_data\` add index \`bucket_data_bucket_op_index\` (\`group_id\`, \`bucket_name\`, \`op_id\`);`); + this.addSql(`alter table \`bucket_data\` add index \`bucket_data_source_index\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + + this.addSql(`create table \`bucket_parameters\` (\`id\` bigint unsigned not null, \`group_id\` int not null, \`source_table\` varchar(255) not null, \`source_key\` varbinary(1024) not null, \`lookup\` varbinary(1024) not null, \`bucket_parameters\` json not null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`bucket_parameters\` add index \`bucket_parameters_lookup_index\` (\`group_id\`, \`lookup\`, \`id\`);`); + this.addSql(`alter table \`bucket_parameters\` add index \`bucket_parameters_source_index\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + + this.addSql(`create table \`current_data\` (\`id\` varchar(255) not null, \`group_id\` int not null, \`source_table\` varchar(255) not null, \`source_key\` varbinary(1024) not null, \`buckets\` json not null, \`lookups\` json not null, \`data\` longblob not null, \`pending_delete\` bigint null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`current_data\` add index \`current_data_source_index\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + this.addSql(`alter table \`current_data\` add index \`current_data_pending_delete_index\` (\`group_id\`, \`pending_delete\`);`); + + this.addSql(`create table \`instance\` (\`id\` varchar(255) not null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + + this.addSql(`create table \`source_tables\` (\`id\` varchar(255) not null, \`group_id\` int not null, \`connection_id\` int not null, \`relation_id\` json null, \`schema_name\` varchar(255) not null, \`table_name\` varchar(255) not null, \`replica_id_columns\` json null, \`snapshot_done\` tinyint(1) not null default true, \`snapshot_total_estimated_count\` bigint null, \`snapshot_replicated_count\` bigint null, \`snapshot_last_key\` blob null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`source_tables\` add index \`source_table_lookup\` (\`group_id\`, \`table_name\`);`); + + this.addSql(`create table \`sync_rules\` (\`id\` int unsigned not null auto_increment primary key, \`state\` varchar(255) not null, \`snapshot_done\` tinyint(1) not null default false, \`snapshot_lsn\` varchar(255) null, \`last_checkpoint\` bigint null, \`last_checkpoint_lsn\` varchar(255) null, \`no_checkpoint_before\` varchar(255) null, \`slot_name\` varchar(255) not null, \`last_checkpoint_ts\` datetime null, \`last_keepalive_ts\` datetime null, \`last_fatal_error\` varchar(255) null, \`last_fatal_error_ts\` datetime null, \`keepalive_op\` bigint null, \`storage_version\` int null, \`content\` longtext not null, \`sync_plan\` json null) default character set utf8mb4 engine = InnoDB;`); + + this.addSql(`create table \`write_checkpoints\` (\`id\` varchar(255) not null, \`sync_rules_id\` int null, \`user_id\` varchar(255) not null, \`checkpoint\` bigint not null, \`heads\` json null, \`created_at\` datetime not null, primary key (\`id\`)) default character set utf8mb4 engine = InnoDB;`); + this.addSql(`alter table \`write_checkpoints\` add index \`write_checkpoints_user_checkpoint_index\` (\`user_id\`, \`sync_rules_id\`, \`checkpoint\`);`); + } + +} diff --git a/modules/module-mikroorm-storage/src/migrations/mysql/Migration20260813000000.ts b/modules/module-mikroorm-storage/src/migrations/mysql/Migration20260813000000.ts new file mode 100644 index 000000000..dac2f903d --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/mysql/Migration20260813000000.ts @@ -0,0 +1,10 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20260813000000 extends Migration { + override up(): void | Promise { + this.addSql(`alter table \`write_checkpoints\` add \`checkpoint_requested_at\` datetime null;`); + this.addSql( + `alter table \`write_checkpoints\` add index \`write_checkpoints_requested_at_index\` (\`checkpoint_requested_at\`);` + ); + } +} diff --git a/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612145654.ts b/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612145654.ts new file mode 100644 index 000000000..8f62fe48e --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260612145654.ts @@ -0,0 +1,29 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20260612145654 extends Migration { + + override up(): void | Promise { + this.addSql(`create table \`bucket_data\` (\`id\` text not null primary key, \`group_id\` integer not null, \`bucket_name\` text not null, \`op_id\` bigint not null, \`op\` text not null, \`source_table\` text null, \`source_key\` blob null, \`table_name\` text null, \`row_id\` text null, \`checksum\` bigint not null, \`data\` text null, \`target_op\` bigint null);`); + this.addSql(`create index \`bucket_data_bucket_op_index\` on \`bucket_data\` (\`group_id\`, \`bucket_name\`, \`op_id\`);`); + this.addSql(`create index \`bucket_data_source_index\` on \`bucket_data\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + + this.addSql(`create table \`bucket_parameters\` (\`id\` bigint not null primary key, \`group_id\` integer not null, \`source_table\` text not null, \`source_key\` blob not null, \`lookup\` blob not null, \`bucket_parameters\` json not null);`); + this.addSql(`create index \`bucket_parameters_lookup_index\` on \`bucket_parameters\` (\`group_id\`, \`lookup\`, \`id\`);`); + this.addSql(`create index \`bucket_parameters_source_index\` on \`bucket_parameters\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + + this.addSql(`create table \`current_data\` (\`id\` text not null primary key, \`group_id\` integer not null, \`source_table\` text not null, \`source_key\` blob not null, \`buckets\` json not null, \`lookups\` json not null, \`data\` blob not null, \`pending_delete\` bigint null);`); + this.addSql(`create index \`current_data_source_index\` on \`current_data\` (\`group_id\`, \`source_table\`, \`source_key\`);`); + this.addSql(`create index \`current_data_pending_delete_index\` on \`current_data\` (\`group_id\`, \`pending_delete\`);`); + + this.addSql(`create table \`instance\` (\`id\` text not null primary key);`); + + this.addSql(`create table \`source_tables\` (\`id\` text not null primary key, \`group_id\` integer not null, \`connection_id\` integer not null, \`relation_id\` json null, \`schema_name\` text not null, \`table_name\` text not null, \`replica_id_columns\` json null, \`snapshot_done\` integer not null default true, \`snapshot_total_estimated_count\` bigint null, \`snapshot_replicated_count\` bigint null, \`snapshot_last_key\` blob null);`); + this.addSql(`create index \`source_table_lookup\` on \`source_tables\` (\`group_id\`, \`table_name\`);`); + + this.addSql(`create table \`sync_rules\` (\`id\` integer not null primary key autoincrement, \`state\` text not null, \`snapshot_done\` integer not null default false, \`snapshot_lsn\` text null, \`last_checkpoint\` bigint null, \`last_checkpoint_lsn\` text null, \`no_checkpoint_before\` text null, \`slot_name\` text not null, \`last_checkpoint_ts\` datetime null, \`last_keepalive_ts\` datetime null, \`last_fatal_error\` text null, \`last_fatal_error_ts\` datetime null, \`keepalive_op\` bigint null, \`storage_version\` integer null, \`content\` text not null, \`sync_plan\` json null);`); + + this.addSql(`create table \`write_checkpoints\` (\`id\` text not null primary key, \`sync_rules_id\` integer null, \`user_id\` text not null, \`checkpoint\` bigint not null, \`heads\` json null, \`created_at\` datetime not null);`); + this.addSql(`create index \`write_checkpoints_user_checkpoint_index\` on \`write_checkpoints\` (\`user_id\`, \`sync_rules_id\`, \`checkpoint\`);`); + } + +} diff --git a/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260813000000.ts b/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260813000000.ts new file mode 100644 index 000000000..27d310e9c --- /dev/null +++ b/modules/module-mikroorm-storage/src/migrations/sqlite/Migration20260813000000.ts @@ -0,0 +1,10 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20260813000000 extends Migration { + override up(): void | Promise { + this.addSql(`alter table \`write_checkpoints\` add column \`checkpoint_requested_at\` datetime null;`); + this.addSql( + `create index \`write_checkpoints_requested_at_index\` on \`write_checkpoints\` (\`checkpoint_requested_at\`);` + ); + } +} diff --git a/modules/module-mikroorm-storage/src/mikro-orm.config.ts b/modules/module-mikroorm-storage/src/mikro-orm.config.ts new file mode 100644 index 000000000..913942429 --- /dev/null +++ b/modules/module-mikroorm-storage/src/mikro-orm.config.ts @@ -0,0 +1,86 @@ +import { resolve } from 'node:path'; +import { mkdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { createMySqlMikroOrmOptions } from './drivers/mysql/mysql-config.js'; +import { createSqliteMikroOrmOptions } from './drivers/sqlite/sqlite-config.js'; +import { + MIKRO_ORM_MYSQL_STORAGE_TYPE, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + normalizeMikroOrmMySqlStorageConfig, + normalizeMikroOrmSqliteStorageConfig +} from './types/types.js'; + +type MikroOrmStorageDialectName = 'sqlite' | 'mysql'; + +const moduleRoot = fileURLToPath(new URL('..', import.meta.url)); + +function getDialect(): MikroOrmStorageDialectName { + const dialect = process.env.MIKRO_ORM_STORAGE_DIALECT ?? 'sqlite'; + if (dialect == 'sqlite') { + return dialect; + } + if (dialect == 'mysql') { + return dialect; + } + + throw new Error(`Unsupported MikroORM storage dialect for migration generation: ${dialect}`); +} + +function sqliteConfig() { + const options = createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: process.env.MIKRO_ORM_SQLITE_DB ?? ':memory:' + }) + ); + const path = resolve(moduleRoot, 'dist/migrations/sqlite'); + const pathTs = resolve(moduleRoot, 'src/migrations/sqlite'); + ensureMigrationDirectories(path, pathTs); + + return { + ...options, + migrations: { + ...options.migrations, + path, + pathTs, + emit: 'ts' + } + }; +} + +function mysqlConfig() { + const options = createMySqlMikroOrmOptions( + normalizeMikroOrmMySqlStorageConfig({ + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: process.env.MIKRO_ORM_MYSQL_URI ?? 'mysql://repl_user:good_password@localhost:3306/powersync' + }) + ); + const path = resolve(moduleRoot, 'dist/migrations/mysql'); + const pathTs = resolve(moduleRoot, 'src/migrations/mysql'); + ensureMigrationDirectories(path, pathTs); + + return { + ...options, + migrations: { + ...options.migrations, + path, + pathTs, + emit: 'ts' + } + }; +} + +function ensureMigrationDirectories(...paths: string[]): void { + for (const path of paths) { + mkdirSync(path, { recursive: true }); + } +} + +const configs = { + sqlite: sqliteConfig, + mysql: mysqlConfig +} satisfies Record unknown>; + +const selectedConfig: unknown = configs[getDialect()](); + +export default selectedConfig; diff --git a/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts b/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts new file mode 100644 index 000000000..edd5908eb --- /dev/null +++ b/modules/module-mikroorm-storage/src/module/MikroOrmStorageModule.ts @@ -0,0 +1,32 @@ +import { modules, system } from '@powersync/service-core'; +import { MikroOrmMigrationAgent } from '../migrations/MikroOrmMigrationAgent.js'; +import { MikroOrmStorageProvider } from '../storage/MikroOrmStorageProvider.js'; +import { + isMikroOrmStorageConfig, + MIKRO_ORM_MYSQL_STORAGE_TYPE, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + MikroOrmStorageConfig +} from '../types/types.js'; + +export class MikroOrmStorageModule extends modules.AbstractModule { + constructor() { + super({ + name: 'MikroORM Bucket Storage' + }); + } + + async initialize(context: system.ServiceContextContainer): Promise { + context.storageEngine.registerProvider(new MikroOrmStorageProvider(MIKRO_ORM_SQLITE_STORAGE_TYPE)); + context.storageEngine.registerProvider(new MikroOrmStorageProvider(MIKRO_ORM_MYSQL_STORAGE_TYPE)); + + if (isMikroOrmStorageConfig(context.configuration.storage)) { + context.migrations.registerMigrationAgent( + new MikroOrmMigrationAgent(MikroOrmStorageConfig.decode(context.configuration.storage)) + ); + } + } + + async teardown(): Promise { + // Teardown for this module is implemented by the storage engine. + } +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts new file mode 100644 index 000000000..efb3a77a6 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketBatch.ts @@ -0,0 +1,679 @@ +import { EntityManager, MikroORM } from '@mikro-orm/core'; +import { BaseObserver, DO_NOT_LOG, Logger, ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { ColumnDescriptor, storage, utils } from '@powersync/service-core'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as uuid from 'uuid'; +import { SourceTable as SourceTableEntity } from '../entities/entities-index.js'; +import { MikroOrmBucketStorageFactory } from './MikroOrmBucketStorageFactory.js'; +import { currentBuckets, currentLookups, MikroOrmPersistedBatch } from './MikroOrmPersistedBatch.js'; +import { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +export interface MikroOrmBucketBatchOptions { + factory: MikroOrmBucketStorageFactory; + orm: MikroORM; + dialect: MikroOrmStorageDialect; + logger: Logger; + syncRules: sync_rules.HydratedSyncConfig; + replicationStreamId: number; + replicationStreamName: string; + lastCheckpointLsn: string | null; + keepaliveOp: bigint | null; + resumeFromLsn: string | null; + storeCurrentData: boolean; + skipExistingRows: boolean; + markRecordUnavailable: storage.BucketStorageMarkRecordUnavailable | undefined; + hooks: storage.StorageHooks | undefined; +} + +const MAX_OPERATION_BATCH_COUNT = 2_000; + +export class MikroOrmBucketBatch + extends BaseObserver + implements storage.BucketStorageBatch +{ + [DO_NOT_LOG] = true; + + public last_flushed_op: bigint | null = null; + public resumeFromLsn: string | null; + public readonly skipExistingRows: boolean; + + private lastCheckpointLsnValue: string | null; + private persistedOp: bigint | null; + private readonly pendingOperations: storage.SaveOptions[] = []; + private readonly customWriteCheckpointBatch: storage.CustomWriteCheckpointOptions[] = []; + private needsActivation = true; + + constructor(private readonly options: MikroOrmBucketBatchOptions) { + super(); + this.lastCheckpointLsnValue = options.lastCheckpointLsn; + this.resumeFromLsn = options.resumeFromLsn; + this.skipExistingRows = options.skipExistingRows; + this.persistedOp = options.keepaliveOp; + } + + get lastCheckpointLsn(): string | null { + return this.lastCheckpointLsnValue; + } + + async [Symbol.asyncDispose](): Promise { + if (this.customWriteCheckpointBatch.length > 0) { + this.options.logger.warn('Disposing writer with unflushed custom write checkpoints'); + } + super.clearListeners(); + } + + async dispose(): Promise { + await this[Symbol.asyncDispose](); + } + + async resolveTables(options: storage.ResolveTablesOptions): Promise { + const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.options.syncRules; + const { connection_id, source } = options; + const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source; + const normalizedReplicaIdColumns = normalizeReplicaIdColumns(replicaIdColumns); + const relationId = { object_id: objectId }; + const em = this.options.orm.em.fork(); + + return em.transactional(async (transactionalEntityManager) => { + const existingRows = await transactionalEntityManager.find(this.options.dialect.sourceTableEntity, { + groupId: this.options.replicationStreamId, + connectionId: connection_id + }); + + let sourceTableRow = + existingRows.find((row) => { + const matchesRelationId = objectId == null || relationObjectId(row.relationId) == objectId; + return ( + row.schemaName == schema && + row.tableName == table && + matchesRelationId && + jsonEquals(row.replicaIdColumns, normalizedReplicaIdColumns) + ); + }) ?? null; + + if (sourceTableRow == null) { + sourceTableRow = transactionalEntityManager.create(this.options.dialect.sourceTableEntity, { + id: options.idGenerator ? String(options.idGenerator()) : uuid.v4(), + groupId: this.options.replicationStreamId, + connectionId: connection_id, + relationId, + schemaName: schema, + tableName: table, + replicaIdColumns: normalizedReplicaIdColumns, + snapshotDone: false, + snapshotTotalEstimatedCount: null, + snapshotReplicatedCount: null, + snapshotLastKey: null + }); + transactionalEntityManager.persist(sourceTableRow); + await transactionalEntityManager.flush(); + } + + const sourceTable = sourceTableFromRow(sourceTableRow, connectionTag, syncRules); + sourceTable.storeCurrentData = sendsCompleteRows !== true; + + const dropTables = existingRows + .filter((row) => row.id != sourceTableRow.id) + .filter((row) => { + const matchesTableName = row.schemaName == schema && row.tableName == table; + return objectId == null ? matchesTableName : relationObjectId(row.relationId) == objectId || matchesTableName; + }) + .map((row) => sourceTableFromRow(row, connectionTag, syncRules)); + + return { + tables: [sourceTable], + dropTables + }; + }); + } + + async getSourceTableStatus(table: storage.SourceTable): Promise { + const row = await this.options.orm.em.fork().findOne(this.options.dialect.sourceTableEntity, { + groupId: this.options.replicationStreamId, + id: String(table.id) + }); + + return row == null ? null : sourceTableFromRow(row, table.ref.connectionTag, this.options.syncRules); + } + + async save(record: storage.SaveOptions): Promise { + const { after, before, sourceTable, tag } = record; + const storeCurrentData = this.options.storeCurrentData && sourceTable.storeCurrentData; + for (const event of this.getTableEvents(sourceTable)) { + this.iterateListeners((cb) => + cb.replicationEvent?.({ + batch: this, + table: sourceTable, + data: { + op: tag, + after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined, + before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined + }, + event + }) + ); + } + + if (!sourceTable.syncData && !sourceTable.syncParameters) { + return null; + } + + this.pendingOperations.push(record); + if (this.pendingOperations.length >= MAX_OPERATION_BATCH_COUNT) { + return this.flush(); + } + return null; + } + + async truncate(sourceTables: storage.SourceTable[]): Promise { + await this.flush(); + + const em = this.options.orm.em.fork(); + let nextOpId = await this.getNextOpId(em); + const firstOpId = nextOpId; + + await em.transactional(async (transactionalEntityManager) => { + const persistedBatch = this.createPersistedBatch(transactionalEntityManager); + for (const table of sourceTables) { + if (!table.syncData && !table.syncParameters) { + continue; + } + + const rows = await transactionalEntityManager.find(this.options.dialect.currentDataEntity, { + groupId: this.options.replicationStreamId, + sourceTable: String(table.id) + }); + + for (const row of rows) { + const sourceKey = storage.deserializeReplicaId(Buffer.from(row.sourceKey)); + if (table.syncData) { + nextOpId = persistedBatch.persistBucketData({ + table, + sourceKey, + existingBuckets: currentBuckets(row), + evaluated: [], + nextOpId + }); + } + + if (table.syncParameters) { + nextOpId = persistedBatch.persistParameterData({ + table, + sourceKey, + existingLookups: currentLookups(row), + evaluated: [], + nextOpId + }); + } + + transactionalEntityManager.remove(row); + } + } + await transactionalEntityManager.flush(); + }); + + if (nextOpId == firstOpId) { + return null; + } + + const lastOpId = nextOpId - 1n; + this.persistedOp = lastOpId; + this.last_flushed_op = lastOpId; + this.logFlushedOperations({ + operationCount: sourceTables.length, + firstOpId, + nextOpId, + operationLabel: 'truncate source table' + }); + return { flushed_op: lastOpId }; + } + + async drop(sourceTables: storage.SourceTable[]): Promise { + const result = await this.truncate(sourceTables); + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + for (const table of sourceTables) { + await transactionalEntityManager.nativeDelete(this.options.dialect.sourceTableEntity, { + groupId: this.options.replicationStreamId, + id: String(table.id) + }); + } + }); + return result; + } + + async flush(_options?: storage.BatchBucketFlushOptions): Promise { + let result: storage.FlushedResult | null = null; + if (this.pendingOperations.length > 0) { + await this.options.hooks?.beforeBatchFlush?.(this); + const operations = this.pendingOperations.splice(0); + const em = this.options.orm.em.fork(); + let nextOpId = await this.getNextOpId(em); + const firstOpId = nextOpId; + for (const batch of chunked(operations, MAX_OPERATION_BATCH_COUNT)) { + await this.options.orm.em.fork().transactional(async (transactionalEntityManager) => { + const persistedBatch = this.createPersistedBatch(transactionalEntityManager); + nextOpId = await persistedBatch.persistOperations(batch, nextOpId); + await transactionalEntityManager.flush(); + }); + } + const lastOpId = nextOpId - 1n; + this.persistedOp = lastOpId; + this.last_flushed_op = lastOpId; + result = { flushed_op: lastOpId }; + this.logFlushedOperations({ + operationCount: operations.length, + firstOpId, + nextOpId + }); + await this.options.hooks?.afterBatchFlush?.(this); + } + + await this.flushCustomWriteCheckpoints(); + return result; + } + + async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { + await this.flush(); + const createEmptyCheckpoints = options?.createEmptyCheckpoints ?? true; + const now = new Date(); + const em = this.options.orm.em.fork(); + + const result = await em.transactional(async (transactionalEntityManager) => { + const syncRulesRow = await transactionalEntityManager.findOneOrFail(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + + const canCheckpoint = + syncRulesRow.snapshotDone === true && + (syncRulesRow.lastCheckpointLsn == null || syncRulesRow.lastCheckpointLsn <= lsn) && + (syncRulesRow.noCheckpointBefore == null || syncRulesRow.noCheckpointBefore <= lsn); + + let checkpointCreated = false; + + if (canCheckpoint) { + const newLastCheckpoint = maxBigint( + syncRulesRow.lastCheckpoint, + this.persistedOp, + syncRulesRow.keepaliveOp, + 0n + ); + const changed = syncRulesRow.lastCheckpoint !== newLastCheckpoint || syncRulesRow.keepaliveOp != null; + + if (changed || createEmptyCheckpoints) { + transactionalEntityManager.assign(syncRulesRow, { + lastCheckpointLsn: lsn, + lastCheckpointTs: now, + lastKeepaliveTs: now, + lastFatalError: null, + keepaliveOp: null, + lastCheckpoint: newLastCheckpoint, + snapshotLsn: null + }); + checkpointCreated = true; + if (newLastCheckpoint != null) { + await transactionalEntityManager.nativeDelete(this.options.dialect.currentDataEntity, { + groupId: this.options.replicationStreamId, + pendingDelete: { $lte: newLastCheckpoint } + }); + } + } else { + transactionalEntityManager.assign(syncRulesRow, { + lastKeepaliveTs: now + }); + } + } else { + transactionalEntityManager.assign(syncRulesRow, { + keepaliveOp: maxBigint(syncRulesRow.keepaliveOp, this.persistedOp, 0n), + lastKeepaliveTs: now + }); + } + + await transactionalEntityManager.flush(); + return { + checkpointBlocked: !canCheckpoint, + checkpointCreated + }; + }); + + if (!result.checkpointBlocked) { + await this.autoActivate(lsn); + } + + this.persistedOp = null; + this.lastCheckpointLsnValue = lsn; + this.options.factory.checkpointWatcher.notify(); + return result; + } + + keepalive(lsn: string): Promise { + return this.commit(lsn, { createEmptyCheckpoints: true }); + } + + async setResumeLsn(lsn: string): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOneOrFail(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + em.assign(row, { snapshotLsn: lsn }); + await em.flush(); + this.resumeFromLsn = lsn; + } + + async markAllSnapshotDone(noCheckpointBeforeLsn: string): Promise { + await this.markSnapshotDoneInternal(noCheckpointBeforeLsn); + } + + async markSnapshotDone(noCheckpointBeforeLsn: string, options?: { throwOnConflict?: boolean }): Promise { + const remaining = await this.options.orm.em.fork().count(this.options.dialect.sourceTableEntity, { + groupId: this.options.replicationStreamId, + snapshotDone: false + }); + + if (remaining > 0) { + if (options?.throwOnConflict ?? true) { + throw new ReplicationAssertionError( + `Cannot mark snapshot done while ${remaining} source table${remaining == 1 ? '' : 's'} still require snapshotting` + ); + } + return; + } + + await this.markSnapshotDoneInternal(noCheckpointBeforeLsn); + } + + async markTableSnapshotRequired(_table: storage.SourceTable): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOneOrFail(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + em.assign(row, { snapshotDone: false }); + await em.flush(); + } + + async markTableSnapshotDone( + tables: storage.SourceTable[], + noCheckpointBeforeLsn?: string + ): Promise { + const ids = tables.map((table) => String(table.id)); + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + const rows = await transactionalEntityManager.find(this.options.dialect.sourceTableEntity, { + id: { $in: ids } + }); + for (const row of rows) { + transactionalEntityManager.assign(row, { + snapshotDone: true, + snapshotTotalEstimatedCount: null, + snapshotReplicatedCount: null, + snapshotLastKey: null + }); + } + + if (noCheckpointBeforeLsn != null) { + await this.assignNoCheckpointBefore(transactionalEntityManager, noCheckpointBeforeLsn); + } + }); + + return tables.map((table) => { + const copy = table.clone(); + copy.snapshotComplete = true; + copy.snapshotStatus = undefined; + return copy; + }); + } + + async updateTableProgress( + table: storage.SourceTable, + progress: Partial + ): Promise { + const copy = table.clone(); + const snapshotStatus = { + totalEstimatedCount: progress.totalEstimatedCount ?? copy.snapshotStatus?.totalEstimatedCount ?? 0, + replicatedCount: progress.replicatedCount ?? copy.snapshotStatus?.replicatedCount ?? 0, + lastKey: progress.lastKey ?? copy.snapshotStatus?.lastKey ?? null + }; + copy.snapshotStatus = snapshotStatus; + + const em = this.options.orm.em.fork(); + const row = await em.findOneOrFail(this.options.dialect.sourceTableEntity, { + id: String(table.id) + }); + em.assign(row, { + snapshotTotalEstimatedCount: BigInt(snapshotStatus.totalEstimatedCount), + snapshotReplicatedCount: BigInt(snapshotStatus.replicatedCount), + snapshotLastKey: snapshotStatus.lastKey + }); + await em.flush(); + + return copy; + } + + addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void { + this.customWriteCheckpointBatch.push({ + ...checkpoint, + sync_rules_id: this.options.replicationStreamId + }); + } + + private async flushCustomWriteCheckpoints(): Promise { + if (this.customWriteCheckpointBatch.length == 0) { + return; + } + + const batch = this.customWriteCheckpointBatch.splice(0); + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + for (const checkpoint of batch) { + const existing = await transactionalEntityManager.findOne(this.options.dialect.writeCheckpointEntity, { + userId: checkpoint.user_id, + syncRulesId: checkpoint.sync_rules_id + }); + + if (existing == null) { + const row = transactionalEntityManager.create(this.options.dialect.writeCheckpointEntity, { + id: uuid.v4(), + syncRulesId: checkpoint.sync_rules_id, + userId: checkpoint.user_id, + checkpoint: checkpoint.checkpoint, + heads: null, + checkpointRequestedAt: checkpoint.checkpoint_requested_at ?? null, + createdAt: new Date() + }); + transactionalEntityManager.persist(row); + } else { + transactionalEntityManager.assign(existing, { + checkpoint: checkpoint.checkpoint, + checkpointRequestedAt: checkpoint.checkpoint_requested_at ?? null, + createdAt: new Date() + }); + } + } + await transactionalEntityManager.flush(); + }); + this.options.factory.checkpointWatcher.notify(); + } + + private createPersistedBatch(transactionalEntityManager: EntityManager): MikroOrmPersistedBatch { + return new MikroOrmPersistedBatch({ + transactionalEntityManager, + dialect: this.options.dialect, + logger: this.options.logger, + syncRules: this.options.syncRules, + replicationStreamId: this.options.replicationStreamId, + storeCurrentData: this.options.storeCurrentData, + skipExistingRows: this.options.skipExistingRows, + markRecordUnavailable: this.options.markRecordUnavailable + }); + } + + private async getNextOpId(em: EntityManager): Promise { + const [bucketData] = await em.find( + this.options.dialect.bucketDataEntity, + {}, + { orderBy: { opId: 'DESC' }, limit: 1 } + ); + const [bucketParameters] = await em.find( + this.options.dialect.bucketParametersEntity, + {}, + { orderBy: { id: 'DESC' }, limit: 1 } + ); + const [pendingDelete] = await em.find( + this.options.dialect.currentDataEntity, + { pendingDelete: { $ne: null } }, + { orderBy: { pendingDelete: 'DESC' }, limit: 1 } + ); + + return maxBigint(bucketData?.opId, bucketParameters?.id, pendingDelete?.pendingDelete, 0n) + 1n; + } + + private logFlushedOperations(options: { + operationCount: number; + firstOpId: bigint; + nextOpId: bigint; + operationLabel?: string; + }): void { + const storageOperationCount = options.nextOpId - options.firstOpId; + const operationLabel = options.operationLabel ?? 'source operation'; + const pluralizedOperationLabel = options.operationCount == 1 ? operationLabel : `${operationLabel}s`; + + if (storageOperationCount == 0n) { + this.options.logger.info( + `[${this.options.replicationStreamName}] Flushed ${options.operationCount} ${pluralizedOperationLabel} to MikroORM storage DB with no new storage ops` + ); + return; + } + + this.options.logger.info( + `[${this.options.replicationStreamName}] Flushed ${options.operationCount} ${pluralizedOperationLabel} to MikroORM storage DB as ${storageOperationCount.toString()} storage ops (${options.firstOpId.toString()}-${(options.nextOpId - 1n).toString()})` + ); + } + + private async markSnapshotDoneInternal(noCheckpointBeforeLsn: string): Promise { + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + await this.assignNoCheckpointBefore(transactionalEntityManager, noCheckpointBeforeLsn); + }); + this.options.factory.checkpointWatcher.notify(); + } + + private async assignNoCheckpointBefore( + transactionalEntityManager: EntityManager, + noCheckpointBeforeLsn: string + ): Promise { + const row = await transactionalEntityManager.findOneOrFail(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + transactionalEntityManager.assign(row, { + snapshotDone: true, + lastKeepaliveTs: new Date(), + noCheckpointBefore: + row.noCheckpointBefore == null || row.noCheckpointBefore < noCheckpointBeforeLsn + ? noCheckpointBeforeLsn + : row.noCheckpointBefore + }); + } + + private async autoActivate(lsn: string): Promise { + if (!this.needsActivation) { + return; + } + + const em = this.options.orm.em.fork(); + let didActivate = false; + await em.transactional(async (transactionalEntityManager) => { + const syncRulesRow = await transactionalEntityManager.findOne(this.options.dialect.syncRulesEntity, { + id: this.options.replicationStreamId + }); + + if (syncRulesRow?.state == storage.SyncRuleState.PROCESSING && syncRulesRow.snapshotDone) { + transactionalEntityManager.assign(syncRulesRow, { state: storage.SyncRuleState.ACTIVE }); + const oldRows = await transactionalEntityManager.find(this.options.dialect.syncRulesEntity, { + id: { $ne: this.options.replicationStreamId }, + state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] } + }); + for (const oldRow of oldRows) { + transactionalEntityManager.assign(oldRow, { state: storage.SyncRuleState.STOP }); + } + await transactionalEntityManager.flush(); + didActivate = true; + this.needsActivation = false; + } else if (syncRulesRow?.state != storage.SyncRuleState.PROCESSING) { + this.needsActivation = false; + } + }); + + if (didActivate) { + this.options.logger.info(`Activated new replication stream at ${lsn}`); + } + } + + private getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] { + return this.options.syncRules.eventDescriptors.filter((event) => + [...event.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref)) + ); + } +} + +function sourceTableFromRow( + row: SourceTableEntity, + connectionTag: string, + syncRules: sync_rules.HydratedSyncConfig +): storage.SourceTable { + const ref = { connectionTag, schema: row.schemaName, name: row.tableName }; + const sourceTable = new storage.SourceTable({ + id: row.id, + ref, + objectId: relationObjectId(row.relationId), + replicaIdColumns: replicaIdColumns(row.replicaIdColumns), + snapshotComplete: row.snapshotDone ?? true, + ...syncRules.getMatchingSources(ref) + }); + + if (!sourceTable.snapshotComplete) { + sourceTable.snapshotStatus = { + totalEstimatedCount: Number(row.snapshotTotalEstimatedCount ?? -1n), + replicatedCount: Number(row.snapshotReplicatedCount ?? 0n), + lastKey: row.snapshotLastKey ?? null + }; + } + + sourceTable.syncEvent = syncRules.tableTriggersEvent(ref); + sourceTable.syncData = sourceTable.bucketDataSources.length > 0; + sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0; + return sourceTable; +} + +function normalizeReplicaIdColumns(replicaIdColumns: ColumnDescriptor[]): ColumnDescriptor[] { + return replicaIdColumns.map((column) => ({ + name: column.name, + type: column.type, + typeId: typeof column.typeId === 'undefined' ? column.typeId : Number(column.typeId) + })); +} + +function replicaIdColumns(value: unknown): ColumnDescriptor[] { + return Array.isArray(value) ? (value as ColumnDescriptor[]) : []; +} + +function relationObjectId(value: unknown): string | number | undefined { + if (value == null || typeof value != 'object' || Array.isArray(value)) { + return undefined; + } + const objectId = (value as Record).object_id; + return typeof objectId == 'string' || typeof objectId == 'number' ? objectId : undefined; +} + +function jsonEquals(left: unknown, right: unknown): boolean { + return JSON.stringify(left ?? null) == JSON.stringify(right ?? null); +} + +function maxBigint(...values: (bigint | null | undefined)[]): bigint { + return values.reduce((max, value) => (value != null && value > max ? value : max), 0n); +} + +function* chunked(items: T[], size: number): Generator { + for (let index = 0; index < items.length; index += size) { + yield items.slice(index, index + size); + } +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts new file mode 100644 index 000000000..54324cf45 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmBucketStorageFactory.ts @@ -0,0 +1,247 @@ +import { MikroORM } from '@mikro-orm/core'; +import { DO_NOT_LOG } from '@powersync/lib-services-framework'; +import { GetIntanceOptions, storage } from '@powersync/service-core'; +import crypto from 'crypto'; +import * as uuid from 'uuid'; +import type { BucketData, BucketParameters, CurrentData, SyncRules } from '../entities/entities-index.js'; +import { MikroOrmPersistedReplicationStream } from './MikroOrmPersistedReplicationStream.js'; +import { MikroOrmCheckpointWatcher, MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; +import { MikroOrmSyncRulesStorage } from './MikroOrmSyncRulesStorage.js'; + +export interface MikroOrmBucketStorageFactoryOptions { + orm: MikroORM; + dialect: MikroOrmStorageDialect; + slotNamePrefix: string; +} + +export class MikroOrmBucketStorageFactory extends storage.BucketStorageFactory { + [DO_NOT_LOG] = true; + + readonly orm: MikroORM; + readonly dialect: MikroOrmStorageDialect; + readonly slotNamePrefix: string; + readonly checkpointWatcher: MikroOrmCheckpointWatcher; + + constructor(options: MikroOrmBucketStorageFactoryOptions) { + super(); + this.orm = options.orm; + this.dialect = options.dialect; + this.slotNamePrefix = options.slotNamePrefix; + this.checkpointWatcher = options.dialect.createCheckpointWatcher(); + } + + async [Symbol.asyncDispose](): Promise { + await this.orm.close(true); + } + + getInstance( + replicationStream: storage.PersistedReplicationStream, + options?: GetIntanceOptions + ): storage.SyncRulesBucketStorage { + const syncRuleStorage = new MikroOrmSyncRulesStorage({ + factory: this, + orm: this.orm, + dialect: this.dialect, + replicationStream + }); + + if (!options?.skipLifecycleHooks) { + this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage)); + } + + syncRuleStorage.registerListener({ + batchStarted: (batch) => { + batch.registerListener({ + replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) + }); + } + }); + + return syncRuleStorage; + } + + async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { + const storageVersion = + options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; + const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion]; + if (storageConfig == null) { + throw new Error(`Unsupported storage version ${storageVersion}`); + } + + const em = this.orm.em.fork(); + let row: SyncRules; + await em.transactional(async (transactionalEntityManager) => { + const processingRows = await transactionalEntityManager.find(this.dialect.syncRulesEntity, { + state: storage.SyncRuleState.PROCESSING + }); + for (const processingRow of processingRows) { + transactionalEntityManager.assign(processingRow, { state: storage.SyncRuleState.STOP }); + } + + const syncRules = transactionalEntityManager.create(this.dialect.syncRulesEntity, { + state: storage.SyncRuleState.PROCESSING, + snapshotDone: false, + snapshotLsn: null, + lastCheckpoint: null, + lastCheckpointLsn: null, + noCheckpointBefore: null, + slotName: this.generateReplicationStreamName(), + lastCheckpointTs: null, + lastKeepaliveTs: null, + lastFatalError: null, + lastFatalErrorTs: null, + keepaliveOp: null, + storageVersion, + content: options.config.yaml, + syncPlan: options.config.plan + }); + + transactionalEntityManager.persist(syncRules); + await transactionalEntityManager.flush(); + row = syncRules; + }); + + return new MikroOrmPersistedReplicationStream(this.orm, this.dialect, row!); + } + + async restartReplication(replicationStreamId: number): Promise { + const active = await this.getActiveSyncConfig(); + const deploying = await this.getDeployingSyncConfig(); + const stream = + deploying?.replicationStream.replicationStreamId == replicationStreamId + ? deploying + : active?.replicationStream.replicationStreamId == replicationStreamId + ? active + : null; + + if (stream == null) { + return; + } + + await this.updateSyncRules(stream.content.asUpdateOptions()); + } + + async getActiveSyncConfig(): Promise { + return this.getSyncConfigForStates([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED]); + } + + async getDeployingSyncConfig(): Promise { + return this.getSyncConfigForStates([storage.SyncRuleState.PROCESSING]); + } + + async getReplicatingReplicationStreams(): Promise { + const rows = await this.findSyncRulesRows([storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]); + return rows.map((row) => new MikroOrmPersistedReplicationStream(this.orm, this.dialect, row)); + } + + async getStoppedReplicationStreams(): Promise { + const rows = await this.findSyncRulesRows([storage.SyncRuleState.STOP]); + return rows.map((row) => new MikroOrmPersistedReplicationStream(this.orm, this.dialect, row)); + } + + async getStorageMetrics(): Promise { + const em = this.orm.em.fork(); + const [operations, parameters, currentData] = await Promise.all([ + em.find(this.dialect.bucketDataEntity, {}), + em.find(this.dialect.bucketParametersEntity, {}), + em.find(this.dialect.currentDataEntity, {}) + ]); + + return { + operations_size_bytes: operations.reduce((total, row) => total + estimateBucketDataSize(row), 0), + parameters_size_bytes: parameters.reduce((total, row) => total + estimateBucketParametersSize(row), 0), + replication_size_bytes: currentData.reduce((total, row) => total + estimateCurrentDataSize(row), 0) + }; + } + + async getPowerSyncInstanceId(): Promise { + const em = this.orm.em.fork(); + const instanceEntity = this.dialect.instanceEntity; + const [existingRow] = await em.find(instanceEntity, {}, { limit: 1 }); + let row = existingRow ?? null; + if (row == null) { + row = em.create(instanceEntity, { id: uuid.v4() }); + await em.persist(row).flush(); + } + return row.id; + } + + async getSystemIdentifier(): Promise { + return { + id: `${this.dialect.type}:${await this.getPowerSyncInstanceId()}`, + type: this.dialect.type + }; + } + + private async getSyncConfigForStates(states: storage.SyncRuleState[]): Promise { + const [row] = await this.findSyncRulesRows(states); + if (row == null) { + return null; + } + + const replicationStream = new MikroOrmPersistedReplicationStream(this.orm, this.dialect, row); + const storageInstance = this.getInstance(replicationStream, { skipLifecycleHooks: true }); + return { + content: replicationStream.syncConfigContent[0], + replicationStream, + storage: storageInstance + }; + } + + private async findSyncRulesRows(states: storage.SyncRuleState[]): Promise { + const em = this.orm.em.fork(); + return await em.find(this.dialect.syncRulesEntity, { state: { $in: states } }, { orderBy: { id: 'DESC' } }); + } + + private generateReplicationStreamName(): string { + return `${this.slotNamePrefix}${Date.now()}_${crypto.randomBytes(2).toString('hex')}`; + } +} + +function estimateString(value: string | null | undefined): number { + return value == null ? 0 : value.length; +} + +function estimateBuffer(value: Buffer | Uint8Array | null | undefined): number { + return value == null ? 0 : value.byteLength; +} + +function estimateJson(value: unknown): number { + return typeof value == 'string' ? value.length : JSON.stringify(value ?? null).length; +} + +function estimateBucketDataSize(row: BucketData): number { + return ( + 80 + + estimateString(row.id) + + estimateString(row.bucketName) + + estimateString(row.op) + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateString(row.tableName) + + estimateString(row.rowId) + + estimateString(row.data) + ); +} + +function estimateBucketParametersSize(row: BucketParameters): number { + return ( + 80 + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateBuffer(row.lookup) + + estimateJson(row.bucketParameters) + ); +} + +function estimateCurrentDataSize(row: CurrentData): number { + return ( + 80 + + estimateString(row.id) + + estimateString(row.sourceTable) + + estimateBuffer(row.sourceKey) + + estimateJson(row.buckets) + + estimateJson(row.lookups) + + estimateBuffer(row.data) + ); +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmCompactor.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmCompactor.ts new file mode 100644 index 000000000..d54ccb670 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmCompactor.ts @@ -0,0 +1,512 @@ +import type { MikroORM } from '@mikro-orm/core'; +import { Logger } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import * as uuid from 'uuid'; +import type { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +interface CurrentBucketState { + bucket: string; + seen: Map; + trackingSize: number; + lastNotPut: bigint | null; + opsSincePut: number; +} + +interface ParameterCompactionRow { + id: bigint; + sourceTable: string; + sourceKey: Buffer; + lookup: Buffer; + bucketParameters: unknown; +} + +interface RawParameterCompactionRow { + id: bigint | number | string; + source_table?: string; + sourceTable?: string; + source_key?: Buffer | Uint8Array; + sourceKey?: Buffer | Uint8Array; + lookup: Buffer | Uint8Array; + bucket_parameters?: unknown; + bucketParameters?: unknown; +} + +interface RawBucketDataRow { + id: string; + bucket_name: string; + op_id: bigint | number | string; + op: string; + source_table: string | null; + source_key: Buffer | Uint8Array | null; + table_name: string | null; + row_id: string | null; + checksum: bigint | number | string; + target_op: bigint | number | string | null; +} + +interface CompactionBucketDataRow { + id: string; + bucketName: string; + opId: bigint; + op: string; + sourceTable: string | null; + sourceKey: Buffer | null; + tableName: string | null; + rowId: string | null; + checksum: bigint; + targetOp: bigint | null; +} + +export interface MikroOrmCompactOptions extends storage.CompactOptions { + logger: Logger; +} + +const BIGINT_MAX = 9223372036854775807n; +const DEFAULT_CLEAR_BATCH_LIMIT = 5000; +const DEFAULT_MOVE_BATCH_LIMIT = 2000; +const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000; +const DEFAULT_MEMORY_LIMIT_MB = 64; +const PARAMETER_DELETE_BATCH_LIMIT = 1000; +const PARAMETER_SCAN_BATCH_LIMIT = 10_000; + +export class MikroOrmCompactor { + private readonly idLimitBytes: number; + private readonly moveBatchLimit: number; + private readonly moveBatchQueryLimit: number; + private readonly clearBatchLimit: number; + private readonly maxOpId: bigint; + private readonly buckets: string[] | undefined; + private readonly deleteCheckpointRequestsBefore: Date | undefined; + private readonly logger: Logger; + + private pendingMoves: { id: string; targetOp: bigint }[] = []; + + constructor( + private readonly orm: MikroORM, + private readonly dialect: MikroOrmStorageDialect, + private readonly groupId: number, + options: MikroOrmCompactOptions + ) { + this.idLimitBytes = (options.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; + this.moveBatchLimit = options.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT; + this.moveBatchQueryLimit = options.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; + this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; + this.maxOpId = options.maxOpId ?? 0n; + this.buckets = options.compactBuckets; + this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore; + this.logger = options.logger; + } + + async compact(): Promise { + await this.deleteOldCheckpointRequests(); + + if (this.maxOpId <= 0n) { + return; + } + + if (this.buckets != null) { + for (const bucket of this.buckets) { + await this.compactSingleBucket(bucket); + } + } else { + await this.compactAllBuckets(); + } + } + + private async deleteOldCheckpointRequests(): Promise { + if (this.deleteCheckpointRequestsBefore == null) { + return; + } + + await this.orm.em.fork().nativeDelete(this.dialect.writeCheckpointEntity, { + checkpointRequestedAt: { $lt: this.deleteCheckpointRequestsBefore } + }); + } + + async compactParameterData(options: storage.CompactOptions): Promise { + if (this.maxOpId <= 0n) { + return; + } + + const lastByKey = new Map(); + const removeIds = new Set(); + const keysWithDuplicateValues = new Set(); + const maxCacheSize = options.compactParameterCacheLimit ?? 10_000; + const flushDeletes = async (force: boolean) => { + if (removeIds.size < PARAMETER_DELETE_BATCH_LIMIT && !(force && removeIds.size > 0)) { + return; + } + + const removedCount = removeIds.size; + for (const id of removeIds) { + await this.executeRun( + ` + DELETE FROM bucket_parameters + WHERE group_id = ? AND id = ? + `, + [this.groupId, id.toString()] + ); + } + removeIds.clear(); + this.logger.info(`Removed ${removedCount} compacted parameter entries`); + }; + + for await (const row of this.streamParameterRows()) { + const key = parameterKey(row); + const previous = lastByKey.get(key); + if (previous != null && sameBucketParameters(previous.bucketParameters, row.bucketParameters)) { + removeIds.add(row.id); + keysWithDuplicateValues.add(key); + } + + if (isEmptyBucketParameters(row.bucketParameters) && row.id < this.maxOpId && !keysWithDuplicateValues.has(key)) { + await flushDeletes(true); + const candidate = lastByKey.get(key); + await this.executeRun( + ` + DELETE FROM bucket_parameters + WHERE group_id = ? + AND lookup = ? + AND source_table = ? + AND source_key = ? + AND id <= ? + `, + [this.groupId, row.lookup, row.sourceTable, row.sourceKey, row.id.toString()] + ); + if (candidate != null && candidate.id <= row.id) { + removeIds.add(candidate.id); + } + removeIds.add(row.id); + lastByKey.delete(key); + } else { + lastByKey.set(key, row); + } + + if (lastByKey.size > maxCacheSize) { + const oldest = lastByKey.keys().next().value; + if (oldest != null) { + lastByKey.delete(oldest); + } + } + + await flushDeletes(false); + } + await flushDeletes(true); + lastByKey.clear(); + } + + private async *streamParameterRows(): AsyncIterable { + let lastRow: ParameterCompactionRow | null = null; + + while (true) { + const params: unknown[] = [this.groupId, this.maxOpId]; + let cursorFilter = ''; + if (lastRow != null) { + cursorFilter = ` + AND ( + lookup > ? + OR (lookup = ? AND source_table > ?) + OR (lookup = ? AND source_table = ? AND source_key > ?) + OR (lookup = ? AND source_table = ? AND source_key = ? AND id > ?) + ) + `; + params.push( + lastRow.lookup, + lastRow.lookup, + lastRow.sourceTable, + lastRow.lookup, + lastRow.sourceTable, + lastRow.sourceKey, + lastRow.lookup, + lastRow.sourceTable, + lastRow.sourceKey, + lastRow.id + ); + } + params.push(PARAMETER_SCAN_BATCH_LIMIT); + + const rows = await this.executeAll( + ` + SELECT id, source_table, source_key, lookup, bucket_parameters + FROM bucket_parameters + WHERE group_id = ? + AND id <= ? + ${cursorFilter} + ORDER BY lookup ASC, source_table ASC, source_key ASC, id ASC + LIMIT ? + `, + params + ); + + if (rows.length == 0) { + return; + } + + for (const rawRow of rows) { + lastRow = rawParameterRow(rawRow); + yield lastRow; + } + } + } + + private async compactAllBuckets(): Promise { + const discoveryBatchSize = 200; + let lastBucket = ''; + + while (true) { + const rows = await this.executeAll<{ bucket_name: string }>( + ` + SELECT DISTINCT bucket_name + FROM bucket_data + WHERE group_id = ? AND bucket_name > ? + ORDER BY bucket_name ASC + LIMIT ? + `, + [this.groupId, lastBucket, discoveryBatchSize] + ); + if (rows.length == 0) { + break; + } + + for (const row of rows) { + await this.compactSingleBucket(row.bucket_name); + } + lastBucket = rows[rows.length - 1].bucket_name; + } + } + + private async compactSingleBucket(bucket: string): Promise { + const currentState: CurrentBucketState = { + bucket, + seen: new Map(), + trackingSize: 0, + lastNotPut: null, + opsSincePut: 0 + }; + let upperOpIdLimit = BIGINT_MAX; + + while (true) { + const batch = ( + await this.executeAll( + ` + SELECT id, bucket_name, op_id, op, source_table, source_key, table_name, row_id, checksum, target_op + FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id < ? + AND op_id <= ? + ORDER BY op_id DESC + LIMIT ? + `, + [this.groupId, bucket, upperOpIdLimit, this.maxOpId, this.moveBatchQueryLimit] + ) + ).map(rawBucketDataRow); + + if (batch.length == 0) { + break; + } + + upperOpIdLimit = batch[batch.length - 1].opId; + + for (const row of batch) { + let isPersistentPut = row.op == 'PUT'; + + if (row.op == 'REMOVE' || row.op == 'PUT') { + const key = compactBucketRowKey(row); + const targetOp = currentState.seen.get(utils.flatstr(key)); + if (targetOp != null) { + isPersistentPut = false; + this.pendingMoves.push({ id: row.id, targetOp }); + } else if (currentState.trackingSize < this.idLimitBytes) { + currentState.seen.set(utils.flatstr(key), row.opId); + currentState.trackingSize += key.length + 140; + } + } + + if (isPersistentPut) { + currentState.lastNotPut = null; + currentState.opsSincePut = 0; + } else if (row.op != 'CLEAR') { + currentState.lastNotPut ??= row.opId; + currentState.opsSincePut += 1; + } + + if (this.pendingMoves.length >= this.moveBatchLimit) { + await this.flushMoves(); + } + } + } + + await this.flushMoves(); + currentState.seen.clear(); + if (currentState.lastNotPut != null && currentState.opsSincePut > 1) { + this.logger.info( + `Inserting CLEAR at ${this.groupId}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations` + ); + await this.clearBucket(currentState.bucket, currentState.lastNotPut); + } + } + + private async flushMoves(): Promise { + if (this.pendingMoves.length == 0) { + return; + } + + const batch = this.pendingMoves.splice(0); + this.logger.info(`Compacting ${batch.length} ops`); + for (const { id, targetOp } of batch) { + await this.executeRun( + ` + UPDATE bucket_data + SET op = 'MOVE', + target_op = ?, + table_name = NULL, + row_id = NULL, + data = NULL, + source_table = NULL, + source_key = NULL + WHERE id = ? + `, + [targetOp, id] + ); + } + } + + private async clearBucket(bucket: string, op: bigint): Promise { + let done = false; + while (!done) { + const operationRows = ( + await this.executeAll( + ` + SELECT id, bucket_name, op_id, op, source_table, source_key, table_name, row_id, checksum, target_op + FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id <= ? + ORDER BY op_id ASC + LIMIT ? + `, + [this.groupId, bucket, op, this.clearBatchLimit] + ) + ).map(rawBucketDataRow); + + let checksum = 0; + let lastOpId: bigint | null = null; + let targetOp: bigint | null = null; + let gotAnOp = false; + + for (const operation of operationRows) { + if (operation.op != 'MOVE' && operation.op != 'REMOVE' && operation.op != 'CLEAR') { + throw new Error(`Unexpected ${operation.op} operation at ${this.groupId}:${bucket}:${operation.opId}`); + } + + checksum = utils.addChecksums(checksum, Number(operation.checksum)); + lastOpId = operation.opId; + if (operation.op != 'CLEAR') { + gotAnOp = true; + } + if (operation.targetOp != null && (targetOp == null || operation.targetOp > targetOp)) { + targetOp = operation.targetOp; + } + } + + if (!gotAnOp || lastOpId == null) { + done = true; + return; + } + + this.logger.info(`Flushing CLEAR at ${lastOpId}`); + await this.executeRun( + ` + DELETE FROM bucket_data + WHERE group_id = ? + AND bucket_name = ? + AND op_id <= ? + `, + [this.groupId, bucket, lastOpId] + ); + await this.executeRun( + ` + INSERT INTO bucket_data ( + id, + group_id, + bucket_name, + op_id, + op, + checksum, + target_op + ) VALUES (?, ?, ?, ?, 'CLEAR', ?, ?) + `, + [uuid.v4(), this.groupId, bucket, lastOpId, BigInt(checksum), targetOp] + ); + } + } + + private async executeAll(sql: string, params: unknown[]): Promise { + return this.orm.em.getConnection().execute(sql, params, 'all'); + } + + private async executeRun(sql: string, params: unknown[]): Promise { + await this.orm.em.getConnection().execute(sql, params, 'run'); + } +} + +function compactBucketRowKey(row: CompactionBucketDataRow): string { + return `${row.tableName}/${row.rowId}/${row.sourceTable}.${row.sourceKey == null ? '' : row.sourceKey.toString('base64')}`; +} + +function rawBucketDataRow(row: RawBucketDataRow): CompactionBucketDataRow { + return { + id: row.id, + bucketName: row.bucket_name, + opId: BigInt(row.op_id), + op: row.op, + sourceTable: row.source_table, + sourceKey: row.source_key == null ? null : Buffer.from(row.source_key), + tableName: row.table_name, + rowId: row.row_id, + checksum: BigInt(row.checksum), + targetOp: row.target_op == null ? null : BigInt(row.target_op) + }; +} + +function rawParameterRow(row: RawParameterCompactionRow): ParameterCompactionRow { + const sourceTable = row.source_table ?? row.sourceTable; + const sourceKey = row.source_key ?? row.sourceKey; + if (sourceTable == null || sourceKey == null) { + throw new Error('Expected parameter compaction row source columns'); + } + + return { + id: BigInt(row.id), + sourceTable, + sourceKey: Buffer.from(sourceKey), + lookup: Buffer.from(row.lookup), + bucketParameters: row.bucket_parameters ?? row.bucketParameters + }; +} + +function parameterKey(row: ParameterCompactionRow): string { + return `${row.lookup.toString('base64')}/${row.sourceTable}/${row.sourceKey.toString('base64')}`; +} + +function normalizedBucketParameters(value: unknown): string { + if (typeof value != 'string') { + return JSON.stringify(value ?? null); + } + + try { + const parsed = JSON.parse(value); + return typeof parsed == 'string' ? parsed : JSON.stringify(parsed); + } catch { + return value; + } +} + +function sameBucketParameters(left: unknown, right: unknown): boolean { + return normalizedBucketParameters(left) == normalizedBucketParameters(right); +} + +function isEmptyBucketParameters(value: unknown): boolean { + return normalizedBucketParameters(value) == '[]'; +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts new file mode 100644 index 000000000..7425a9095 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedBatch.ts @@ -0,0 +1,419 @@ +import type { EntityManager } from '@mikro-orm/core'; +import type { Logger } from '@powersync/lib-services-framework'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as uuid from 'uuid'; +import type { CurrentData } from '../entities/entities-index.js'; +import type { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +export interface MikroOrmPersistedBatchOptions { + transactionalEntityManager: EntityManager; + dialect: MikroOrmStorageDialect; + logger: Logger; + syncRules: sync_rules.HydratedSyncConfig; + replicationStreamId: number; + storeCurrentData: boolean; + skipExistingRows: boolean; + markRecordUnavailable: storage.BucketStorageMarkRecordUnavailable | undefined; +} + +export type CurrentBucket = { + bucket: string; + table: string; + id: string; +}; + +const MAX_ROW_SIZE = 15 * 1024 * 1024; + +/** + * Handles the writes for a single persisted operation chunk. + * + * `MikroOrmBucketBatch` owns the public batch lifecycle, checkpointing, and + * listener hooks. This class owns the transactional write state for bucket + * data, parameter rows, and current data so large flushes can be split into + * smaller persisted chunks without keeping every tracked entity in one unit of + * work. + */ +export class MikroOrmPersistedBatch { + private readonly currentDataById = new Map(); + + constructor(private readonly options: MikroOrmPersistedBatchOptions) {} + + async persistOperations(operations: storage.SaveOptions[], nextOpId: bigint): Promise { + await this.loadCurrentData(operations); + for (const operation of operations) { + nextOpId = await this.persistOperation(operation, nextOpId); + } + return nextOpId; + } + + persistBucketData(options: { + table: storage.SourceTable; + sourceKey: storage.ReplicaId; + existingBuckets: CurrentBucket[]; + evaluated: sync_rules.EvaluatedRow[]; + nextOpId: bigint; + }): bigint { + const remainingBuckets = new Map(options.existingBuckets.map((bucket) => [currentBucketKey(bucket), bucket])); + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + const deleteChecksum = utils.hashDelete(replicaIdToSubkey(options.table.id, options.sourceKey)); + let nextOpId = options.nextOpId; + + for (const row of options.evaluated) { + remainingBuckets.delete(currentBucketKey(row)); + const data = JSONBig.stringify(row.data); + const checksum = utils.hashData(row.table, row.id, data); + this.options.transactionalEntityManager.persist( + this.options.transactionalEntityManager.create(this.options.dialect.bucketDataEntity, { + id: uuid.v4(), + groupId: this.options.replicationStreamId, + bucketName: row.bucket, + opId: nextOpId++, + op: 'PUT', + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + tableName: row.table, + rowId: row.id, + checksum: BigInt(checksum), + data, + targetOp: null + }) + ); + } + + for (const bucket of remainingBuckets.values()) { + this.options.transactionalEntityManager.persist( + this.options.transactionalEntityManager.create(this.options.dialect.bucketDataEntity, { + id: uuid.v4(), + groupId: this.options.replicationStreamId, + bucketName: bucket.bucket, + opId: nextOpId++, + op: 'REMOVE', + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + tableName: bucket.table, + rowId: bucket.id, + checksum: BigInt(deleteChecksum), + data: null, + targetOp: null + }) + ); + } + + return nextOpId; + } + + persistParameterData(options: { + table: storage.SourceTable; + sourceKey: storage.ReplicaId; + existingLookups: Buffer[]; + evaluated: sync_rules.EvaluatedParameters[]; + nextOpId: bigint; + }): bigint { + const remainingLookups = new Map(options.existingLookups.map((lookup) => [lookup.toString('base64'), lookup])); + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + let nextOpId = options.nextOpId; + + for (const row of options.evaluated) { + const lookup = storage.serializeLookupBuffer(row.lookup); + remainingLookups.delete(lookup.toString('base64')); + this.options.transactionalEntityManager.persist( + this.options.transactionalEntityManager.create(this.options.dialect.bucketParametersEntity, { + id: nextOpId++, + groupId: this.options.replicationStreamId, + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + lookup, + bucketParameters: JSONBig.stringify(row.bucketParameters) + }) + ); + } + + for (const lookup of remainingLookups.values()) { + this.options.transactionalEntityManager.persist( + this.options.transactionalEntityManager.create(this.options.dialect.bucketParametersEntity, { + id: nextOpId++, + groupId: this.options.replicationStreamId, + sourceTable: String(options.table.id), + sourceKey: serializedSourceKey, + lookup, + bucketParameters: '[]' + }) + ); + } + + return nextOpId; + } + + private async persistOperation(record: storage.SaveOptions, nextOpId: bigint): Promise { + const sourceTable = record.sourceTable; + const tableId = String(sourceTable.id); + const afterId = record.afterReplicaId ?? null; + const beforeId = record.beforeReplicaId ?? record.afterReplicaId; + const serializedBeforeId = storage.serializeReplicaId(beforeId); + const existingCurrentDataId = currentDataId(this.options.replicationStreamId, tableId, serializedBeforeId); + const existingCurrentData = this.currentDataById.get(existingCurrentDataId) ?? null; + + const storeCurrentData = this.options.storeCurrentData && sourceTable.storeCurrentData; + let existingBuckets = currentBuckets(existingCurrentData); + let existingLookups = currentLookups(existingCurrentData); + let after: sync_rules.ToastableSqliteRow | null | undefined = record.after; + + if (this.options.skipExistingRows) { + if (record.tag == storage.SaveOperationTag.INSERT) { + if (existingCurrentData != null) { + return nextOpId; + } + } else { + throw new ReplicationAssertionError(`${record.tag} not supported with skipExistingRows: true`); + } + } + + if (record.tag == storage.SaveOperationTag.UPDATE) { + if (existingCurrentData != null && storeCurrentData) { + after = storage.mergeToast(record.after, storage.deserializeBson(Buffer.from(existingCurrentData.data))); + } else if (existingCurrentData == null && storeCurrentData) { + this.options.markRecordUnavailable?.(record); + } + } + + if (beforeId != null && (afterId == null || !storage.replicaIdEquals(beforeId, afterId))) { + if (sourceTable.syncData) { + nextOpId = this.persistBucketData({ + table: sourceTable, + sourceKey: beforeId, + existingBuckets, + evaluated: [], + nextOpId + }); + existingBuckets = []; + } + + if (sourceTable.syncParameters) { + nextOpId = this.persistParameterData({ + table: sourceTable, + sourceKey: beforeId, + existingLookups, + evaluated: [], + nextOpId + }); + existingLookups = []; + } + } + + let newBuckets: CurrentBucket[] = []; + let newLookups: Buffer[] = []; + let afterData: Buffer | undefined; + let afterDataWasTruncated = false; + if (afterId != null && after != null && utils.isCompleteRow(storeCurrentData, after)) { + if (storeCurrentData) { + const prepared = this.serializeCurrentData(record, after); + after = prepared.after; + afterData = prepared.data; + afterDataWasTruncated = prepared.truncated; + } else { + afterData = storage.serializeBson({}); + } + + if (sourceTable.syncData) { + const { results: rawResults, errors } = this.options.syncRules.evaluateRowWithErrors({ + record: after as sync_rules.SqliteRow, + sourceTable: sourceTable.ref, + bucketDataSources: sourceTable.bucketDataSources + }); + const results = afterDataWasTruncated ? rawResults.filter(hasUsableObjectId) : rawResults; + for (const error of errors) { + this.options.logger.error( + `Failed to evaluate data query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + nextOpId = this.persistBucketData({ + table: sourceTable, + sourceKey: afterId, + existingBuckets, + evaluated: results, + nextOpId + }); + newBuckets = results.map((row) => ({ + bucket: row.bucket, + table: row.table, + id: row.id + })); + } + + if (sourceTable.syncParameters) { + const { results, errors } = this.options.syncRules.evaluateParameterRowWithErrors( + sourceTable.ref, + after as sync_rules.SqliteRow, + { + parameterLookupSources: sourceTable.parameterLookupSources + } + ); + for (const error of errors) { + this.options.logger.error( + `Failed to evaluate parameter query on ${sourceTable.qualifiedName}.${after.id}: ${error.error}` + ); + } + nextOpId = this.persistParameterData({ + table: sourceTable, + sourceKey: afterId, + existingLookups, + evaluated: results, + nextOpId + }); + newLookups = results.map((row) => storage.serializeLookupBuffer(row.lookup)); + } + } + + if (afterId != null && afterData != null) { + await this.upsertCurrentData({ + tableId, + sourceKey: afterId, + buckets: newBuckets, + lookups: newLookups, + data: afterData, + pendingDelete: null + }); + } + + if (afterId == null || !storage.replicaIdEquals(beforeId, afterId)) { + nextOpId = await this.deleteCurrentData(tableId, beforeId, nextOpId); + } + + return nextOpId; + } + + private async loadCurrentData(operations: storage.SaveOptions[]): Promise { + const ids = new Set(); + for (const operation of operations) { + const tableId = String(operation.sourceTable.id); + const beforeId = operation.beforeReplicaId ?? operation.afterReplicaId; + ids.add(currentDataId(this.options.replicationStreamId, tableId, storage.serializeReplicaId(beforeId))); + + const afterId = operation.afterReplicaId ?? null; + if (afterId != null) { + ids.add(currentDataId(this.options.replicationStreamId, tableId, storage.serializeReplicaId(afterId))); + } + } + + const rows = + ids.size == 0 + ? [] + : await this.options.transactionalEntityManager.find(this.options.dialect.currentDataEntity, { + id: { $in: [...ids] } + }); + for (const row of rows) { + this.currentDataById.set(row.id, row); + } + } + + private serializeCurrentData( + record: storage.SaveOptions, + after: sync_rules.ToastableSqliteRow + ): { after: sync_rules.ToastableSqliteRow; data: Buffer; truncated: boolean } { + try { + const serialized = storage.serializeBson(after); + if (serialized.byteLength > MAX_ROW_SIZE) { + throw new Error(`Row too large: ${serialized.byteLength}`); + } + return { after, data: serialized, truncated: false }; + } catch (e) { + const error = e instanceof Error ? e : new Error(String(e)); + this.options.logger.warn( + `Data too big on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.message}` + ); + + // Keep the current_data row present, but drop field values. This mirrors + // the Postgres storage behavior for oversized BSON payloads and allows + // future TOAST-style updates to be marked unavailable instead of crashing + // the replication batch. + const emptyValues = Object.fromEntries(Object.keys(after).map((key) => [key, undefined])); + return { after: emptyValues, data: storage.serializeBson(emptyValues), truncated: true }; + } + } + + private async upsertCurrentData(options: { + tableId: string; + sourceKey: storage.ReplicaId; + buckets: CurrentBucket[]; + lookups: Buffer[]; + data: Buffer; + pendingDelete: bigint | null; + }): Promise { + const serializedSourceKey = storage.serializeReplicaId(options.sourceKey); + const id = currentDataId(this.options.replicationStreamId, options.tableId, serializedSourceKey); + const payload = { + id, + groupId: this.options.replicationStreamId, + sourceTable: options.tableId, + sourceKey: serializedSourceKey, + buckets: options.buckets, + lookups: options.lookups.map((lookup) => lookup.toString('hex')), + data: options.data, + pendingDelete: options.pendingDelete + }; + const existing = this.currentDataById.get(id) ?? null; + if (existing == null) { + const row = this.options.transactionalEntityManager.create(this.options.dialect.currentDataEntity, payload); + this.options.transactionalEntityManager.persist(row); + this.currentDataById.set(id, row); + } else { + this.options.transactionalEntityManager.assign(existing, payload); + this.currentDataById.set(id, existing); + } + } + + private async deleteCurrentData(tableId: string, sourceKey: storage.ReplicaId, nextOpId: bigint): Promise { + const serializedSourceKey = storage.serializeReplicaId(sourceKey); + const id = currentDataId(this.options.replicationStreamId, tableId, serializedSourceKey); + const payload = { + id, + groupId: this.options.replicationStreamId, + sourceTable: tableId, + sourceKey: serializedSourceKey, + buckets: [], + lookups: [], + data: storage.serializeBson({}), + pendingDelete: nextOpId + }; + const existing = this.currentDataById.get(id) ?? null; + if (existing == null) { + const row = this.options.transactionalEntityManager.create(this.options.dialect.currentDataEntity, payload); + this.options.transactionalEntityManager.persist(row); + this.currentDataById.set(id, row); + } else { + this.options.transactionalEntityManager.assign(existing, payload); + this.currentDataById.set(id, existing); + } + return nextOpId + 1n; + } +} + +export function currentBuckets(row: CurrentData | null): CurrentBucket[] { + return Array.isArray(row?.buckets) ? (row.buckets as CurrentBucket[]) : []; +} + +export function currentLookups(row: CurrentData | null): Buffer[] { + return Array.isArray(row?.lookups) ? (row.lookups as string[]).map((lookup) => Buffer.from(lookup, 'hex')) : []; +} + +function currentDataId(groupId: number, sourceTable: string, sourceKey: Buffer): string { + return `${groupId}:${sourceTable}:${sourceKey.toString('hex')}`; +} + +function currentBucketKey(bucket: CurrentBucket | sync_rules.EvaluatedRow): string { + return `${bucket.bucket}/${bucket.table}/${bucket.id}`; +} + +function hasUsableObjectId(row: sync_rules.EvaluatedRow): boolean { + return row.id !== '' || row.data.id != null; +} + +function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string { + if (storage.isUUID(id)) { + return `${tableId}/${id.toHexString()}`; + } + return uuid.v5(storage.serializeBson({ table: tableId, id }), utils.ID_NAMESPACE); +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts new file mode 100644 index 000000000..5db87c451 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmPersistedReplicationStream.ts @@ -0,0 +1,82 @@ +import { MikroORM } from '@mikro-orm/core'; +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import type { SyncRules } from '../entities/entities-index.js'; +import { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +export class MikroOrmPersistedSyncConfigContent extends storage.PersistedSyncConfigContent { + constructor( + private readonly orm: MikroORM, + private readonly dialect: MikroOrmStorageDialect, + row: SyncRules + ) { + super({ + replicationStreamId: Number(row.id), + sync_rules_content: row.content, + compiled_plan: row.syncPlan, + replicationStreamName: row.slotName, + storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION, + syncConfigId: String(row.id), + syncConfigState: row.state as storage.SyncRuleState + }); + } + + async getSyncConfigStatus(): Promise { + const em = this.orm.em.fork(); + const row = await em.findOne(this.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + + return row == null ? null : syncConfigStatusFromRow(row); + } +} + +export class MikroOrmPersistedReplicationStream extends storage.PersistedReplicationStream { + current_lock: storage.ReplicationLock | null = null; + readonly syncConfigContent: readonly MikroOrmPersistedSyncConfigContent[]; + + constructor( + private readonly orm: MikroORM, + private readonly dialect: MikroOrmStorageDialect, + private readonly row: SyncRules + ) { + super({ + replicationStreamId: Number(row.id), + replicationStreamName: row.slotName, + state: row.state, + storageVersion: row.storageVersion ?? storage.LEGACY_STORAGE_VERSION + }); + this.syncConfigContent = [new MikroOrmPersistedSyncConfigContent(this.orm, this.dialect, this.row)]; + } + + parsed(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet { + return this.syncConfigContent[0].parsed(options); + } + + async lock(): Promise { + if (this.current_lock != null) { + throw new ServiceError(ErrorCode.PSYNC_S1003, `Replication stream is locked by this process.`); + } + + return (this.current_lock = { + sync_rules_id: this.replicationStreamId, + release: async () => { + this.current_lock = null; + } + }); + } +} + +export function syncConfigStatusFromRow(row: SyncRules): storage.PersistedSyncConfigStatus { + return { + id: String(row.id), + replicationStreamId: Number(row.id), + state: row.state, + snapshot_done: row.snapshotDone, + last_checkpoint_lsn: row.lastCheckpointLsn, + last_fatal_error: row.lastFatalError, + last_fatal_error_ts: row.lastFatalErrorTs, + last_keepalive_ts: row.lastKeepaliveTs, + last_checkpoint_ts: row.lastCheckpointTs + }; +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmReportStorage.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmReportStorage.ts new file mode 100644 index 000000000..ccd623e8a --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmReportStorage.ts @@ -0,0 +1,34 @@ +import { storage } from '@powersync/service-core'; +import { event_types } from '@powersync/service-types'; + +export class MikroOrmReportStorage implements storage.ReportStorage { + async [Symbol.asyncDispose](): Promise { + // Report storage is intentionally a no-op in the initial MikroORM storage slice. + } + + async reportClientConnection(_data: event_types.ClientConnectionBucketData): Promise {} + + async reportClientDisconnection(_data: event_types.ClientDisconnectionEventData): Promise {} + + async getConnectedClients(): Promise { + return { users: [], sdks: [] } as unknown as event_types.ClientConnectionReportResponse; + } + + async getClientConnectionReports( + _data: event_types.ClientConnectionReportRequest + ): Promise { + return { users: [], sdks: [] } as unknown as event_types.ClientConnectionReportResponse; + } + + async getGeneralClientConnectionAnalytics( + _data: event_types.ClientConnectionAnalyticsRequest + ): Promise> { + return { + items: [], + count: 0, + more: false + } as unknown as event_types.PaginatedResponse; + } + + async deleteOldConnectionData(_data: event_types.DeleteOldConnectionData): Promise {} +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts new file mode 100644 index 000000000..c51a8d658 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageDialect.ts @@ -0,0 +1,101 @@ +import type { EntityClass, EntityManager, EntityName } from '@mikro-orm/core'; +import type { BucketDataRequest } from '@powersync/service-core'; +import type { + BucketData, + BucketParameters, + CurrentData, + Instance, + SourceTable, + SyncRules, + WriteCheckpoint +} from '../entities/entities-index.js'; + +/** + * Complete database-specific adapter consumed by the common MikroORM storage classes. + */ +export interface MikroOrmStorageDialect { + /** Public storage type identifier, e.g. `mikroorm:sqlite`. */ + readonly type: string; + /** Flat entity class list passed to MikroORM initialization and migration tooling. */ + readonly entityClasses: EntityName[]; + /** Stored bucket operations, typically read by bucket and operation id. */ + readonly bucketDataEntity: EntityClass; + /** Materialized parameter lookups for bucket parameter queries. */ + readonly bucketParametersEntity: EntityClass; + /** Latest known source row state used by replication batching and compaction paths. */ + readonly currentDataEntity: EntityClass; + /** Singleton service instance identity row. */ + readonly instanceEntity: EntityClass; + /** Source table replication metadata and snapshot progress. */ + readonly sourceTableEntity: EntityClass; + /** Sync rule versions and replication stream state. */ + readonly syncRulesEntity: EntityClass; + /** User write checkpoint rows. */ + readonly writeCheckpointEntity: EntityClass; + /** + * Stream bucket operation rows for sync reads. + * + * Hot bucket reads need to avoid buffering a whole result set before chunking. SQL drivers can implement this with + * MikroORM query builder streaming, while non-SQL drivers can provide their closest cursor equivalent. + */ + streamBucketDataRows(options: MikroOrmBucketDataStreamOptions): AsyncIterable; + /** Create a checkpoint watcher suitable for this database's notification capabilities. */ + createCheckpointWatcher(): MikroOrmCheckpointWatcher; +} + +export interface MikroOrmBucketDataStreamOptions { + readonly em: EntityManager; + readonly groupId: number; + readonly checkpoint: bigint; + readonly dataBuckets: BucketDataRequest[]; + readonly limit: number; +} + +/** + * Minimal checkpoint notification abstraction. + * + * SQLite can only notify listeners inside the current process, while future drivers can map this onto database-level + * notifications that work across service instances. + */ +export interface MikroOrmCheckpointWatcher { + /** Wake local watchers after this process writes a relevant checkpoint. */ + notify(): void; + /** Yield whenever a checkpoint change should be re-read. */ + watch(signal: AbortSignal): AsyncIterable; +} + +/** + * Process-local checkpoint watcher for drivers without a database notification implementation. + * + * This is sufficient for single-process runners and tests. Drivers that support cross-process notifications should + * replace this with a watcher backed by the database or an external event channel. + */ +export class InProcessMikroOrmCheckpointWatcher implements MikroOrmCheckpointWatcher { + private readonly listeners = new Set<() => void>(); + + notify(): void { + for (const listener of this.listeners) { + listener(); + } + } + + async *watch(signal: AbortSignal): AsyncIterable { + while (!signal.aborted) { + yield await new Promise((resolve) => { + const listener = () => { + this.listeners.delete(listener); + resolve(); + }; + this.listeners.add(listener); + signal.addEventListener( + 'abort', + () => { + this.listeners.delete(listener); + resolve(); + }, + { once: true } + ); + }); + } + } +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts new file mode 100644 index 000000000..fecc29f2d --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmStorageProvider.ts @@ -0,0 +1,110 @@ +import { logger } from '@powersync/lib-services-framework'; +import { storage, system } from '@powersync/service-core'; +import { createMySqlMikroOrm } from '../drivers/mysql/mysql-config.js'; +import { createMySqlMikroOrmStorageFactory } from '../drivers/mysql/MySqlMikroOrmStorageFactory.js'; +import { createSqliteMikroOrm } from '../drivers/sqlite/sqlite-config.js'; +import { createSqliteMikroOrmStorageFactory } from '../drivers/sqlite/SqliteMikroOrmStorageFactory.js'; +import { + isMikroOrmMySqlStorageConfig, + isMikroOrmSqliteStorageConfig, + MIKRO_ORM_MYSQL_STORAGE_TYPE, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + MikroOrmMySqlStorageConfig, + MikroOrmSqliteStorageConfig, + normalizeMikroOrmMySqlStorageConfig, + normalizeMikroOrmSqliteStorageConfig +} from '../types/types.js'; +import { MikroOrmReportStorage } from './MikroOrmReportStorage.js'; + +export class MikroOrmStorageProvider implements storage.StorageProvider { + constructor(private readonly storageType: typeof MIKRO_ORM_SQLITE_STORAGE_TYPE | typeof MIKRO_ORM_MYSQL_STORAGE_TYPE) {} + + get type() { + return this.storageType; + } + + async getStorage(options: storage.GetStorageOptions): Promise { + const { storage: storageConfig } = options.resolvedConfig; + + if (this.storageType == MIKRO_ORM_SQLITE_STORAGE_TYPE && isMikroOrmSqliteStorageConfig(storageConfig)) { + assertSqliteServiceMode(options.serviceMode); + + const decodedConfig = MikroOrmSqliteStorageConfig.decode(storageConfig); + const normalizedConfig = normalizeMikroOrmSqliteStorageConfig(decodedConfig); + const orm = await createSqliteMikroOrm(normalizedConfig); + await orm.schema.update(); + + const storageFactory = await createSqliteMikroOrmStorageFactory({ + config: normalizedConfig, + slotNamePrefix: options.resolvedConfig.slot_name_prefix, + orm + }); + + return activeStorage({ + storageFactory, + tearDownLabel: `MikroORM SQLite storage: ${normalizedConfig.filename}`, + tearDown: () => orm.schema.drop() + }); + } + + if (this.storageType == MIKRO_ORM_MYSQL_STORAGE_TYPE && isMikroOrmMySqlStorageConfig(storageConfig)) { + const decodedConfig = MikroOrmMySqlStorageConfig.decode(storageConfig); + const normalizedConfig = normalizeMikroOrmMySqlStorageConfig(decodedConfig); + const orm = await createMySqlMikroOrm(normalizedConfig); + await orm.schema.update(); + + const storageFactory = await createMySqlMikroOrmStorageFactory({ + config: normalizedConfig, + slotNamePrefix: options.resolvedConfig.slot_name_prefix, + orm + }); + + return activeStorage({ + storageFactory, + tearDownLabel: 'MikroORM MySQL storage', + tearDown: () => orm.schema.drop() + }); + } + + throw new Error(`Cannot create ${this.storageType} storage with provided config ${storageConfig.type}`); + } +} + +function activeStorage(options: { + storageFactory: storage.BucketStorageFactory; + tearDownLabel: string; + tearDown: () => Promise; +}): storage.ActiveStorage { + const reportStorage = new MikroOrmReportStorage(); + return { + reportStorage, + storage: options.storageFactory, + shutDown: async () => { + await options.storageFactory[Symbol.asyncDispose](); + }, + tearDown: async () => { + logger.info(`Tearing down ${options.tearDownLabel}...`); + await options.tearDown(); + await options.storageFactory[Symbol.asyncDispose](); + return true; + } + }; +} + +const SQLITE_ALLOWED_COMMAND_MODES = new Set([ + system.ServiceContextMode.COMPACT, + system.ServiceContextMode.TEARDOWN, + system.ServiceContextMode.TEST_CONNECTION +]); + +function assertSqliteServiceMode(serviceMode: string): void { + if (serviceMode == system.ServiceContextMode.UNIFIED || SQLITE_ALLOWED_COMMAND_MODES.has(serviceMode)) { + return; + } + + throw new Error( + `MikroORM SQLite storage only supports the unified service runner. ` + + `SQLite checkpoint notifications are process-local, so split "${system.ServiceContextMode.API}" and "${system.ServiceContextMode.SYNC}" runners cannot safely share this storage. ` + + `Start the service with runner type "${system.ServiceContextMode.UNIFIED}" instead of "${serviceMode}".` + ); +} diff --git a/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts b/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts new file mode 100644 index 000000000..85251bd47 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/MikroOrmSyncRulesStorage.ts @@ -0,0 +1,625 @@ +import { MikroORM } from '@mikro-orm/core'; +import { BaseObserver, DO_NOT_LOG, errors, Logger } from '@powersync/lib-services-framework'; +import { + BucketChecksumRequest, + BucketDataBatchOptions, + BucketDataRequest, + CHECKPOINT_INVALIDATE_ALL, + CheckpointChanges, + GetCheckpointChangesOptions, + PopulateChecksumCacheOptions, + PopulateChecksumCacheResults, + ReplicationCheckpoint, + storage, + StorageCheckpointUpdate, + SyncBucketDataChunk, + utils, + WatchWriteCheckpointOptions +} from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as uuid from 'uuid'; +import type { BucketData } from '../entities/entities-index.js'; +import { MikroOrmBucketBatch } from './MikroOrmBucketBatch.js'; +import { MikroOrmBucketStorageFactory } from './MikroOrmBucketStorageFactory.js'; +import { MikroOrmCompactor } from './MikroOrmCompactor.js'; +import { MikroOrmStorageDialect } from './MikroOrmStorageDialect.js'; + +export interface MikroOrmSyncRulesStorageOptions { + factory: MikroOrmBucketStorageFactory; + orm: MikroORM; + dialect: MikroOrmStorageDialect; + replicationStream: storage.PersistedReplicationStream; +} + +export class MikroOrmSyncRulesStorage + extends BaseObserver + implements storage.SyncRulesBucketStorage +{ + [DO_NOT_LOG] = true; + + readonly replicationStreamId: number; + readonly replicationStreamName: string; + readonly storageConfig: storage.StorageVersionConfig; + readonly factory: MikroOrmBucketStorageFactory; + readonly logger: Logger; + + private readonly parsedSyncConfigSets = new Map(); + + private writeCheckpointModeValue = storage.WriteCheckpointMode.MANAGED; + + constructor(private readonly options: MikroOrmSyncRulesStorageOptions) { + super(); + this.replicationStreamId = options.replicationStream.replicationStreamId; + this.replicationStreamName = options.replicationStream.replicationStreamName; + this.storageConfig = options.replicationStream.getStorageConfig(); + this.factory = options.factory; + this.logger = options.replicationStream.logger; + } + + get writeCheckpointMode(): storage.WriteCheckpointMode { + return this.writeCheckpointModeValue; + } + + setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void { + this.writeCheckpointModeValue = mode; + } + + async createManagedWriteCheckpoints( + checkpoints: storage.ManagedWriteCheckpointOptions[] + ): Promise { + if (this.writeCheckpointMode !== storage.WriteCheckpointMode.MANAGED) { + throw new errors.ValidationError( + `Attempting to create a managed Write Checkpoint when the current Write Checkpoint mode is set to "${this.writeCheckpointMode}"` + ); + } + + const uniqueCheckpoints = storage.uniqueManagedWriteCheckpoints(checkpoints); + if (uniqueCheckpoints.length == 0) { + return { writeCheckpoints: new Map(), shouldAdvance: false }; + } + const writeCheckpoints = new Map(); + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + for (const checkpoint of uniqueCheckpoints) { + const [latest] = await transactionalEntityManager.find( + this.options.dialect.writeCheckpointEntity, + { + userId: checkpoint.user_id, + syncRulesId: null + }, + { + orderBy: { checkpoint: 'DESC' }, + limit: 1 + } + ); + + const requestedCheckpoint = checkpoint.checkpoint_request_id; + if (requestedCheckpoint != null && latest != null && requestedCheckpoint <= latest.checkpoint) { + if (requestedCheckpoint == latest.checkpoint) { + transactionalEntityManager.assign(latest, { checkpointRequestedAt: new Date() }); + } + writeCheckpoints.set(checkpoint.user_id, latest.checkpoint); + continue; + } + + if (requestedCheckpoint != null && latest != null) { + // A newer client-supplied id replaces the managed mapping. Keeping an + // older row would allow that old id to be acknowledged while the new + // request's source head is still pending. + await transactionalEntityManager.nativeDelete(this.options.dialect.writeCheckpointEntity, { + userId: checkpoint.user_id, + syncRulesId: null + }); + } + + const value = requestedCheckpoint ?? (latest?.checkpoint ?? 0n) + 1n; + const row = transactionalEntityManager.create(this.options.dialect.writeCheckpointEntity, { + id: uuid.v4(), + syncRulesId: null, + userId: checkpoint.user_id, + checkpoint: value, + heads: checkpoint.heads, + checkpointRequestedAt: requestedCheckpoint == null ? null : new Date(), + createdAt: new Date() + }); + transactionalEntityManager.persist(row); + writeCheckpoints.set(checkpoint.user_id, value); + } + await transactionalEntityManager.flush(); + }); + + this.factory.checkpointWatcher.notify(); + // This storage does not track whether each managed checkpoint has already + // been processed, so conservatively force the source marker for every + // matched request. This also makes stale retries recover a lost marker. + return { writeCheckpoints, shouldAdvance: true }; + } + + async lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise { + switch (this.writeCheckpointMode) { + case storage.WriteCheckpointMode.CUSTOM: + return this.lastCustomWriteCheckpoint({ + user_id: filters.user_id, + sync_rules_id: this.replicationStreamId + }); + case storage.WriteCheckpointMode.MANAGED: + if (!('heads' in filters)) { + throw new errors.ValidationError(`Replication HEAD is required for managed Write Checkpoint filtering`); + } + return this.lastManagedWriteCheckpoint(filters); + } + } + + async createWriter(options: storage.CreateWriterOptions): Promise { + const em = this.options.orm.em.fork(); + const syncRules = await em.findOne(this.options.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + + const checkpointLsn = syncRules?.lastCheckpointLsn ?? null; + const writer = new MikroOrmBucketBatch({ + factory: this.factory, + orm: this.options.orm, + dialect: this.options.dialect, + logger: options.logger ?? this.logger, + syncRules: this.getParsedSyncRules(options), + replicationStreamId: this.replicationStreamId, + replicationStreamName: this.replicationStreamName, + lastCheckpointLsn: checkpointLsn, + keepaliveOp: syncRules?.keepaliveOp ?? null, + resumeFromLsn: utils.maxLsn(syncRules?.snapshotLsn, checkpointLsn), + storeCurrentData: options.storeCurrentData, + skipExistingRows: options.skipExistingRows ?? false, + markRecordUnavailable: options.markRecordUnavailable, + hooks: options.hooks + }); + this.iterateListeners((cb) => cb.batchStarted?.(writer)); + return writer; + } + + async startBatch( + options: storage.CreateWriterOptions, + callback: (batch: storage.BucketStorageBatch) => Promise + ): Promise { + await using writer = await this.createWriter(options); + await callback(writer); + await writer.flush(); + return writer.last_flushed_op != null ? { flushed_op: writer.last_flushed_op } : null; + } + + getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): storage.ParsedSyncConfigSet { + let parsed = this.parsedSyncConfigSets.get(options.defaultSchema); + if (parsed == null) { + parsed = this.options.replicationStream.parsed(options); + this.parsedSyncConfigSets.set(options.defaultSchema, parsed); + } + return parsed; + } + + getParsedSyncRules(options: storage.ParseSyncConfigOptions): sync_rules.HydratedSyncConfig { + return this.getParsedSyncConfigSet(options).hydratedSyncConfig; + } + + async terminate(options?: storage.TerminateOptions): Promise { + if (!options || options.clearStorage) { + await this.clear(options); + } + + const em = this.options.orm.em.fork(); + const row = await em.findOne(this.options.dialect.syncRulesEntity, { id: this.replicationStreamId }); + if (row != null) { + em.assign(row, { + state: storage.SyncRuleState.TERMINATED, + snapshotDone: false + }); + await em.flush(); + } + this.factory.checkpointWatcher.notify(); + } + + async getStatus(): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOne(this.options.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + + if (row == null) { + throw new Error('Cannot find replication stream status'); + } + return { + snapshotDone: row.snapshotDone && row.lastCheckpointLsn != null, + resumeLsn: utils.maxLsn(row.snapshotLsn, row.lastCheckpointLsn) + }; + } + + async clear(_options?: storage.ClearStorageOptions): Promise { + const em = this.options.orm.em.fork(); + await em.transactional(async (transactionalEntityManager) => { + const row = await transactionalEntityManager.findOne(this.options.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + if (row != null) { + transactionalEntityManager.assign(row, { + snapshotDone: false, + lastCheckpointLsn: null, + lastCheckpoint: null, + noCheckpointBefore: null + }); + } + + await transactionalEntityManager.nativeDelete(this.options.dialect.bucketDataEntity, { + groupId: this.replicationStreamId + }); + await transactionalEntityManager.nativeDelete(this.options.dialect.bucketParametersEntity, { + groupId: this.replicationStreamId + }); + await transactionalEntityManager.nativeDelete(this.options.dialect.currentDataEntity, { + groupId: this.replicationStreamId + }); + await transactionalEntityManager.nativeDelete(this.options.dialect.sourceTableEntity, { + groupId: this.replicationStreamId + }); + }); + + this.factory.checkpointWatcher.notify(); + } + + async reportError(e: any): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOne(this.options.dialect.syncRulesEntity, { id: this.replicationStreamId }); + if (row != null) { + em.assign(row, { + lastFatalError: String(e.message ?? 'Replication failure'), + lastFatalErrorTs: new Date() + }); + await em.flush(); + } + } + + async compact(options?: storage.CompactOptions): Promise { + let maxOpId = options?.maxOpId; + if (maxOpId == null) { + const checkpoint = await this.getCheckpoint(); + maxOpId = checkpoint.checkpoint; + } + + const compactor = new MikroOrmCompactor(this.options.orm, this.options.dialect, this.replicationStreamId, { + ...options, + maxOpId, + logger: this.logger + }); + await compactor.compact(); + + if (options?.compactParameterData) { + await compactor.compactParameterData(options); + } + } + + async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise { + return { buckets: 0 }; + } + + async getCheckpoint(): Promise { + const em = this.options.orm.em.fork(); + const row = await em.findOne(this.options.dialect.syncRulesEntity, { + id: this.replicationStreamId + }); + + return { + checkpoint: row?.lastCheckpoint ?? 0n, + lsn: row?.lastCheckpointLsn ?? null, + getParameterSets: (lookups, limit) => this.getParameterSets(row?.lastCheckpoint ?? 0n, lookups, limit) + }; + } + + async getCheckpointChanges(_options: GetCheckpointChangesOptions): Promise { + return CHECKPOINT_INVALIDATE_ALL; + } + + async *watchCheckpointChanges(options: WatchWriteCheckpointOptions): AsyncIterable { + let lastCheckpoint: bigint | null = null; + let lastCheckpointLsn: string | null = null; + let lastWriteCheckpoint: bigint | null = null; + const { signal, user_id } = options; + + const watcher = this.factory.checkpointWatcher.watch(signal)[Symbol.asyncIterator](); + let nextNotification: Promise> | null = null; + let readImmediately = true; + + try { + while (!signal.aborted) { + if (!readImmediately) { + nextNotification ??= watcher.next(); + const result = await nextNotification; + nextNotification = null; + if (result.done) { + return; + } + } + readImmediately = false; + + if (signal.aborted) { + return; + } + + const base = await this.getCheckpoint(); + const currentWriteCheckpoint = await this.lastWriteCheckpoint({ + user_id, + heads: base.lsn == null ? {} : { '1': base.lsn } + }); + + if ( + currentWriteCheckpoint == lastWriteCheckpoint && + base.checkpoint == lastCheckpoint && + base.lsn == lastCheckpointLsn + ) { + continue; + } + + lastWriteCheckpoint = currentWriteCheckpoint; + lastCheckpoint = base.checkpoint; + lastCheckpointLsn = base.lsn; + nextNotification = watcher.next(); + + yield { + base, + writeCheckpoint: currentWriteCheckpoint, + update: CHECKPOINT_INVALIDATE_ALL + }; + } + } finally { + await watcher.return?.(); + } + } + + async *getBucketDataBatch( + checkpoint: ReplicationCheckpoint, + dataBuckets: BucketDataRequest[], + options?: BucketDataBatchOptions + ): AsyncIterable { + if (dataBuckets.length == 0) { + return; + } + + const batchRowLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT; + const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES; + const startOpByBucket = new Map(dataBuckets.map((request) => [request.bucket, request.start])); + const rows = this.options.dialect.streamBucketDataRows({ + em: this.options.orm.em.fork(), + groupId: this.replicationStreamId, + checkpoint: checkpoint.checkpoint, + dataBuckets, + limit: batchRowLimit + }); + + let chunkSizeBytes = 0; + let currentChunk: utils.SyncBucketData | null = null; + let targetOp: bigint | null = null; + let batchRowCount = 0; + + for await (const row of rows) { + const rowSizeBytes = row.data?.length ?? 0; + const sizeExceeded = + chunkSizeBytes >= chunkSizeLimitBytes || + ((currentChunk?.data.length ?? 0) > 0 && chunkSizeBytes + rowSizeBytes > chunkSizeLimitBytes) || + (currentChunk?.data.length ?? 0) >= batchRowLimit; + + if (currentChunk == null || currentChunk.bucket != row.bucketName || sizeExceeded) { + let start: string | undefined = undefined; + if (currentChunk != null) { + if (currentChunk.bucket == row.bucketName) { + currentChunk.has_more = true; + start = currentChunk.next_after; + } + + const yieldChunk = currentChunk; + currentChunk = null; + chunkSizeBytes = 0; + yield { chunkData: yieldChunk, targetOp }; + targetOp = null; + if (batchRowCount >= batchRowLimit) { + break; + } + } + + if (start == null) { + const startOpId = startOpByBucket.get(row.bucketName); + if (startOpId == null) { + throw new Error(`data for unexpected bucket: ${row.bucketName}`); + } + start = utils.internalToExternalOpId(startOpId); + } + currentChunk = { + bucket: row.bucketName, + after: start, + has_more: false, + data: [], + next_after: start + }; + } + + const entry = bucketDataRowToOpEntry(row); + currentChunk.data.push(entry); + currentChunk.next_after = entry.op_id; + if (row.targetOp != null && (targetOp == null || row.targetOp > targetOp)) { + targetOp = row.targetOp; + } + + chunkSizeBytes += rowSizeBytes; + batchRowCount++; + } + + if (currentChunk != null) { + currentChunk.has_more = batchRowCount >= batchRowLimit; + yield { chunkData: currentChunk, targetOp }; + } + } + + async getChecksums(checkpoint: ReplicationCheckpoint, buckets: BucketChecksumRequest[]): Promise { + const result: utils.ChecksumMap = new Map(); + for (const bucket of buckets) { + const rows = await this.options.orm.em.fork().find( + this.options.dialect.bucketDataEntity, + { + groupId: this.replicationStreamId, + bucketName: bucket.bucket, + opId: { $lte: checkpoint.checkpoint } + }, + { + orderBy: { opId: 'ASC' } + } + ); + const entries = rows.map(bucketDataRowToOpEntry); + const checksum = entries.reduce((total, entry) => utils.addChecksums(total, Number(entry.checksum)), 0); + result.set(bucket.bucket, { + bucket: bucket.bucket, + checksum, + count: entries.length + }); + } + return result; + } + + clearChecksumCache(): void { + // No checksum cache exists in the initial MikroORM storage slice. + } + + private async lastCustomWriteCheckpoint(filters: storage.CustomWriteCheckpointFilters): Promise { + const row = await this.options.orm.em.fork().findOne( + this.options.dialect.writeCheckpointEntity, + { + userId: filters.user_id, + syncRulesId: filters.sync_rules_id + }, + { + orderBy: { checkpoint: 'DESC' } + } + ); + return row?.checkpoint ?? null; + } + + private async getParameterSets( + checkpoint: bigint, + lookups: sync_rules.ScopedParameterLookup[], + limit: number + ): Promise { + const resultsByLookup = new Map(); + let totalRows = 0; + + for (const lookup of lookups) { + const serializedLookup = storage.serializeLookupBuffer(lookup); + const rows = await this.options.orm.em.fork().find( + this.options.dialect.bucketParametersEntity, + { + groupId: this.replicationStreamId, + lookup: serializedLookup, + id: { $lte: checkpoint } + }, + { + orderBy: { id: 'DESC' } + } + ); + + const latestBySource = new Map(); + for (const row of rows) { + const key = `${row.sourceTable}:${Buffer.from(row.sourceKey).toString('hex')}`; + if (!latestBySource.has(key)) { + latestBySource.set(key, row); + } + } + + for (const row of latestBySource.values()) { + const parameterRows = parseBucketParameters(row.bucketParameters); + if (parameterRows.length == 0) { + continue; + } + totalRows += parameterRows.length; + if (totalRows > limit) { + throw new storage.ParameterSetLimitExceededError(limit); + } + const existing = resultsByLookup.get(lookup); + if (existing != null) { + existing.push(...parameterRows); + } else { + resultsByLookup.set(lookup, parameterRows); + } + } + } + + const results: sync_rules.ParameterLookupRows[] = []; + resultsByLookup.forEach((rows, lookup) => results.push({ lookup, rows })); + return results; + } + + private async lastManagedWriteCheckpoint(filters: storage.ManagedWriteCheckpointFilters): Promise { + const lsn = filters.heads['1']; + if (lsn == null) { + return null; + } + + const rows = await this.options.orm.em.fork().find( + this.options.dialect.writeCheckpointEntity, + { + userId: filters.user_id, + syncRulesId: null + }, + { + orderBy: { checkpoint: 'DESC' } + } + ); + + return ( + rows.find((row) => { + const rowHead = getPrimaryReplicationHead(row.heads); + return rowHead != null && rowHead <= lsn; + })?.checkpoint ?? null + ); + } +} + +function getPrimaryReplicationHead(heads: unknown): string | null { + if (heads == null || typeof heads != 'object' || Array.isArray(heads)) { + return null; + } + + const head = (heads as Record)['1']; + return typeof head == 'string' ? head : null; +} + +function parseBucketParameters(value: unknown): sync_rules.SqliteJsonRow[] { + if (typeof value == 'string') { + return JSONBig.parse(value) as sync_rules.SqliteJsonRow[]; + } + return Array.isArray(value) ? (value as sync_rules.SqliteJsonRow[]) : []; +} + +function bucketDataRowToOpEntry(row: BucketData): utils.OplogEntry { + if (row.op == 'PUT' || row.op == 'REMOVE') { + return { + op_id: utils.internalToExternalOpId(row.opId), + op: row.op, + object_type: row.tableName ?? undefined, + object_id: row.rowId ?? undefined, + checksum: Number(row.checksum), + subkey: + row.sourceTable != null && row.sourceKey != null + ? replicaIdToSubkey(row.sourceTable, storage.deserializeReplicaId(Buffer.from(row.sourceKey))) + : undefined, + data: row.op == 'REMOVE' ? null : (row.data ?? undefined) + }; + } + + return { + op_id: utils.internalToExternalOpId(row.opId), + op: row.op as 'CLEAR' | 'MOVE', + checksum: Number(row.checksum) + }; +} + +function replicaIdToSubkey(tableId: storage.SourceTableId, id: storage.ReplicaId): string { + if (storage.isUUID(id)) { + return `${tableId}/${id.toHexString()}`; + } + return uuid.v5(storage.serializeBson({ table: tableId, id }), utils.ID_NAMESPACE); +} diff --git a/modules/module-mikroorm-storage/src/storage/storage-index.ts b/modules/module-mikroorm-storage/src/storage/storage-index.ts new file mode 100644 index 000000000..fa6d1b339 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/storage-index.ts @@ -0,0 +1,8 @@ +export * from './MikroOrmBucketBatch.js'; +export * from './MikroOrmBucketStorageFactory.js'; +export * from './MikroOrmPersistedBatch.js'; +export * from './MikroOrmPersistedReplicationStream.js'; +export * from './MikroOrmReportStorage.js'; +export * from './MikroOrmStorageDialect.js'; +export * from './MikroOrmStorageProvider.js'; +export * from './MikroOrmSyncRulesStorage.js'; diff --git a/modules/module-mikroorm-storage/src/storage/unsupported.ts b/modules/module-mikroorm-storage/src/storage/unsupported.ts new file mode 100644 index 000000000..9b00ceb16 --- /dev/null +++ b/modules/module-mikroorm-storage/src/storage/unsupported.ts @@ -0,0 +1,3 @@ +export function unsupportedMikroOrmStorageFeature(feature: string): never { + throw new Error(`MikroORM bucket storage does not implement ${feature} yet.`); +} diff --git a/modules/module-mikroorm-storage/src/types/types.ts b/modules/module-mikroorm-storage/src/types/types.ts new file mode 100644 index 000000000..ca087ac5b --- /dev/null +++ b/modules/module-mikroorm-storage/src/types/types.ts @@ -0,0 +1,83 @@ +import { configFile } from '@powersync/service-types'; +import * as t from 'ts-codec'; + +export const MIKRO_ORM_SQLITE_STORAGE_TYPE = 'mikroorm:sqlite'; +export const MIKRO_ORM_MYSQL_STORAGE_TYPE = 'mikroorm:mysql'; + +export const MikroOrmSqliteStorageConfig = configFile.BaseStorageConfig.and( + t.object({ + type: t.literal(MIKRO_ORM_SQLITE_STORAGE_TYPE), + /** + * SQLite database file. Use ":memory:" for in-memory storage. + */ + filename: t.string + }) +); + +export type MikroOrmSqliteStorageConfig = t.Encoded; +export type MikroOrmSqliteStorageConfigDecoded = t.Decoded; + +export const MikroOrmMySqlStorageConfig = configFile.BaseStorageConfig.and( + t.object({ + type: t.literal(MIKRO_ORM_MYSQL_STORAGE_TYPE), + /** + * MySQL connection URI, for example mysql://root:password@localhost:3306/powersync_storage. + */ + uri: t.string + }) +); + +export type MikroOrmMySqlStorageConfig = t.Encoded; +export type MikroOrmMySqlStorageConfigDecoded = t.Decoded; + +export const MikroOrmStorageConfig = MikroOrmSqliteStorageConfig.or(MikroOrmMySqlStorageConfig); +export type MikroOrmStorageConfig = t.Encoded; +export type MikroOrmStorageConfigDecoded = t.Decoded; + +export const isMikroOrmSqliteStorageConfig = ( + config: configFile.GenericStorageConfig +): config is MikroOrmSqliteStorageConfig => { + return config.type == MIKRO_ORM_SQLITE_STORAGE_TYPE; +}; + +export const isMikroOrmMySqlStorageConfig = ( + config: configFile.GenericStorageConfig +): config is MikroOrmMySqlStorageConfig => { + return config.type == MIKRO_ORM_MYSQL_STORAGE_TYPE; +}; + +export const isMikroOrmStorageConfig = (config: configFile.GenericStorageConfig): config is MikroOrmStorageConfig => { + return isMikroOrmSqliteStorageConfig(config) || isMikroOrmMySqlStorageConfig(config); +}; + +export interface NormalizedMikroOrmSqliteStorageConfig { + type: typeof MIKRO_ORM_SQLITE_STORAGE_TYPE; + filename: string; + max_pool_size: number; +} + +export interface NormalizedMikroOrmMySqlStorageConfig { + type: typeof MIKRO_ORM_MYSQL_STORAGE_TYPE; + uri: string; + max_pool_size: number; +} + +export function normalizeMikroOrmSqliteStorageConfig( + config: MikroOrmSqliteStorageConfigDecoded +): NormalizedMikroOrmSqliteStorageConfig { + return { + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: config.filename, + max_pool_size: config.max_pool_size ?? 10 + }; +} + +export function normalizeMikroOrmMySqlStorageConfig( + config: MikroOrmMySqlStorageConfigDecoded +): NormalizedMikroOrmMySqlStorageConfig { + return { + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: config.uri, + max_pool_size: config.max_pool_size ?? 10 + }; +} diff --git a/modules/module-mikroorm-storage/test/src/__snapshots__/mysql-storage.test.ts.snap b/modules/module-mikroorm-storage/test/src/__snapshots__/mysql-storage.test.ts.snap new file mode 100644 index 000000000..cdaac777d --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/__snapshots__/mysql-storage.test.ts.snap @@ -0,0 +1,124 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v1 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v1 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v2 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v2 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v3 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Compaction - v3 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Data - v1 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Data - v2 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`MikroORM MySQL Sync Bucket Storage > Data - v3 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; diff --git a/modules/module-mikroorm-storage/test/src/__snapshots__/storage.test.ts.snap b/modules/module-mikroorm-storage/test/src/__snapshots__/storage.test.ts.snap new file mode 100644 index 000000000..316577eaf --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/__snapshots__/storage.test.ts.snap @@ -0,0 +1,124 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v1 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v1 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v2 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v2 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v3 > partial checksums after compacting (2) 1`] = ` +{ + "bucket": "1#global[]", + "checksum": 1196713877, + "count": 1, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Compaction - v3 > partial checksums after compacting 1`] = ` +{ + "bucket": "1#global[]", + "checksum": -134691003, + "count": 4, +} +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Data - v1 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Data - v2 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; + +exports[`MikroORM SQLite Sync Bucket Storage - Data - v3 > (insert, delete, insert), (delete) 1`] = ` +[ + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, + { + "checksum": 2871785649, + "object_id": "test1", + "op": "PUT", + }, + { + "checksum": 2872534815, + "object_id": "test1", + "op": "REMOVE", + }, +] +`; diff --git a/modules/module-mikroorm-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-mikroorm-storage/test/src/__snapshots__/storage_sync.test.ts.snap new file mode 100644 index 000000000..3c1a48a99 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -0,0 +1,3568 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`sync - MikroORM SQLite > storage v1 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v1 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v2 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > can override priority when subscribing to stream 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#todos|0["a"]", + "checksum": -1712802421, + "count": 1, + "priority": 0, + "subscriptions": [ + { + "sub": 0, + }, + { + "sub": 1, + }, + ], + }, + { + "bucket": "1#todos|0["b"]", + "checksum": -1291414318, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "sub": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": false, + "name": "todos", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["a"]", + "data": [ + { + "checksum": 2582164875, + "data": "{"id":"a","description":"Test 1"}", + "object_id": "a", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 0, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#todos|0["b"]", + "data": [ + { + "checksum": 3003552978, + "data": "{"id":"b","description":"Test 2"}", + "object_id": "b", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > compacting data - invalidate checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > compacting data - invalidate checkpoint 2`] = ` +[ + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": -93886621, + "op": "CLEAR", + "op_id": "2", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 499012468, + "count": 3, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "2", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 1859363232, + "data": "{"id":"t1","description":"Test 1b"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "3", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3028503153, + "data": "{"id":"t2","description":"Test 2b"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "4", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "4", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > encodes sync rules id in buckets for streams 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#test|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#test|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > encodes sync rules id in buckets for streams 2`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "2#test2|0[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "test2", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "2#test2|0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "8a5f3fdd-3f59-5153-92ae-ac115c458441", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > expired token 1`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > expiring token 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > expiring token 2`] = ` +[ + { + "token_expires_in": 0, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sends checkpoint complete line for empty checkpoint 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -1221282404, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3073684892, + "data": "{"id":"t1","description":"sync"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + null, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [], + "write_checkpoint": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync buckets in order 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0[]", + "checksum": 920318466, + "count": 1, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1382098757, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": [ + { + "checksum": 2912868539, + "data": "{"id":"earlier","description":"Test 2"}", + "object_id": "earlier", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "243b0e26-87b2-578a-993c-5ac5b6f7fd64", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "2", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "2", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#b0a[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": -659831575, + "count": 2000, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": -1096116670, + "count": 1, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "last_op_id": "4001", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "b0a", + }, + { + "errors": [], + "is_default": true, + "name": "b0b", + }, + { + "errors": [], + "is_default": true, + "name": "b1", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "1", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4001", + "priority": 1, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "2000", + }, + }, + { + "data": { + "after": "2000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": true, + "next_after": "4000", + }, + }, + { + "checkpoint_diff": { + "last_op_id": "4004", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#b0a[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + { + "bucket": "1#b0b[]", + "checksum": 883076828, + "count": 2001, + "priority": 2, + "subscriptions": [ + { + "default": 1, + }, + ], + }, + { + "bucket": "1#b1[]", + "checksum": 1841937527, + "count": 2, + "priority": 1, + "subscriptions": [ + { + "default": 2, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#b1[]", + "data": undefined, + "has_more": false, + "next_after": "4002", + }, + }, + { + "partial_checkpoint_complete": { + "last_op_id": "4004", + "priority": 1, + }, + }, + { + "data": { + "after": "4000", + "bucket": "1#b0a[]", + "data": undefined, + "has_more": false, + "next_after": "4003", + }, + }, + { + "data": { + "after": "0", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "1999", + }, + }, + { + "data": { + "after": "1999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": true, + "next_after": "3999", + }, + }, + { + "data": { + "after": "3999", + "bucket": "1#b0b[]", + "data": undefined, + "has_more": false, + "next_after": "4004", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "4004", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync legacy non-raw data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -852817836, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3442149460n, + "data": { + "description": "Test +"string"", + "id": "t1", + "large_num": 12345678901234567890n, + }, + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to data query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "1", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to data query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "2", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to global data 1`] = ` +[ + { + "checkpoint": { + "buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "mybucket", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to global data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": 920318466, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 920318466, + "data": "{"id":"t1","description":"Test 1"}", + "object_id": "t1", + "object_type": "test", + "op": "PUT", + "op_id": "1", + "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to global data 3`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#mybucket[]", + "checksum": -93886621, + "count": 2, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "1", + "bucket": "1#mybucket[]", + "data": [ + { + "checksum": 3280762209, + "data": "{"id":"t2","description":"Test 2"}", + "object_id": "t2", + "object_type": "test", + "op": "PUT", + "op_id": "2", + "subkey": "a17e6883-d5d2-599d-a805-d60528127dbd", + }, + ], + "has_more": false, + "next_after": "2", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to parameter query + data 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to parameter query + data 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "2", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 1418351250, + "count": 1, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "data": { + "after": "0", + "bucket": "1#by_user["user1"]", + "data": [ + { + "checksum": 1418351250, + "data": "{"id":"list1","user_id":"user1","name":"User 1"}", + "object_id": "list1", + "object_type": "lists", + "op": "PUT", + "op_id": "1", + "subkey": "b9f16d58-e6f5-55b5-9622-7bc360dba34f", + }, + ], + "has_more": false, + "next_after": "1", + }, + }, + { + "checkpoint_complete": { + "last_op_id": "2", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to parameter query only 1`] = ` +[ + { + "checkpoint": { + "buckets": [], + "last_op_id": "0", + "streams": [ + { + "errors": [], + "is_default": true, + "name": "by_user", + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "0", + }, + }, +] +`; + +exports[`sync - MikroORM SQLite > storage v3 > sync updates to parameter query only 2`] = ` +[ + { + "checkpoint_diff": { + "last_op_id": "1", + "removed_buckets": [], + "updated_buckets": [ + { + "bucket": "1#by_user["user1"]", + "checksum": 0, + "count": 0, + "priority": 3, + "subscriptions": [ + { + "default": 0, + }, + ], + }, + ], + "write_checkpoint": undefined, + }, + }, + { + "checkpoint_complete": { + "last_op_id": "1", + }, + }, +] +`; diff --git a/modules/module-mikroorm-storage/test/src/env.ts b/modules/module-mikroorm-storage/test/src/env.ts new file mode 100644 index 000000000..ac22dc7f0 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/env.ts @@ -0,0 +1,7 @@ +import { utils } from '@powersync/lib-services-framework'; + +export const env = utils.collectEnvironmentVariables({ + MIKROORM_MYSQL_STORAGE_TEST_URI: utils.type.string.default( + process.env.MYSQL_TEST_URI ?? 'mysql://repl_user:good_password@localhost:3306/powersync' + ) +}); diff --git a/modules/module-mikroorm-storage/test/src/migrations.test.ts b/modules/module-mikroorm-storage/test/src/migrations.test.ts new file mode 100644 index 000000000..6f2e9cc6c --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/migrations.test.ts @@ -0,0 +1,463 @@ +import { rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { MikroORM } from '@mikro-orm/core'; +import { MySqlDriver } from '@mikro-orm/mysql'; +import { SqliteDriver } from '@mikro-orm/sqlite'; +import { Direction } from '@powersync/lib-services-framework'; +import { storage } from '@powersync/service-core'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createMySqlMikroOrmOptions } from '../../src/drivers/mysql/mysql-config.js'; +import { createSqliteMikroOrmOptions } from '../../src/drivers/sqlite/sqlite-config.js'; +import { sqliteMikroOrmStorageDialect } from '../../src/drivers/sqlite/sqlite-dialect.js'; +import { MIKRO_ORM_MYSQL_STORAGE_TYPE, MIKRO_ORM_SQLITE_STORAGE_TYPE } from '../../src/index.js'; +import { MikroOrmMigrationAgent } from '../../src/migrations/MikroOrmMigrationAgent.js'; +import { normalizeMikroOrmMySqlStorageConfig, normalizeMikroOrmSqliteStorageConfig } from '../../src/types/types.js'; +import { env } from './env.js'; + +describe('MikroORM migrations', () => { + const dbFiles: string[] = []; + + const createDbFile = () => { + const filename = join(tmpdir(), `powersync-mikroorm-storage-${process.pid}-${Date.now()}-${dbFiles.length}.sqlite`); + dbFiles.push(filename); + return filename; + }; + + afterEach(async () => { + await Promise.all( + dbFiles + .splice(0) + .flatMap((file) => [ + rm(file, { force: true }), + rm(`${file}-shm`, { force: true }), + rm(`${file}-wal`, { force: true }) + ]) + ); + }); + + it('enables WAL and read replicas for file-backed SQLite storage', async () => { + const filename = createDbFile(); + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename, + max_pool_size: 3 + }) + ) + ); + + try { + const driver = orm.em.getDriver(); + const writeConnection = driver.getConnection('write'); + const readConnection = driver.getConnection('read'); + + expect(readConnection).not.toBe(writeConnection); + await expect(writeConnection.execute<{ journal_mode: string }[]>('PRAGMA journal_mode')).resolves.toEqual([ + { journal_mode: 'wal' } + ]); + await expect(readConnection.execute<{ journal_mode: string }[]>('PRAGMA journal_mode')).resolves.toEqual([ + { journal_mode: 'wal' } + ]); + } finally { + await orm.close(true); + } + }); + + it('reads from a replica while the write connection has an open transaction', async () => { + const filename = createDbFile(); + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename, + max_pool_size: 2 + }) + ) + ); + + try { + await orm.em.execute('create table concurrency_test (id integer primary key, value text)', [], 'run'); + await orm.em.execute(`insert into concurrency_test (id, value) values (1, 'committed')`, [], 'run'); + + let releaseWriter!: () => void; + const releaseWriterPromise = new Promise((resolve) => { + releaseWriter = resolve; + }); + let writerReady!: () => void; + const writerReadyPromise = new Promise((resolve) => { + writerReady = resolve; + }); + + const writer = orm.em.fork().transactional(async (transactionalEntityManager) => { + await transactionalEntityManager.execute( + `update concurrency_test set value = 'uncommitted' where id = 1`, + [], + 'run' + ); + writerReady(); + await releaseWriterPromise; + }); + + await writerReadyPromise; + try { + const start = performance.now(); + const rows = await orm.em + .fork() + .getKysely<{ concurrency_test: { id: number; value: string } }>({ type: 'read' }) + .selectFrom('concurrency_test') + .select('value') + .where('id', '=', 1) + .execute(); + const elapsed = performance.now() - start; + + expect(rows).toEqual([{ value: 'committed' }]); + expect(elapsed).toBeLessThan(1_000); + } finally { + releaseWriter(); + await writer; + } + } finally { + await orm.close(true); + } + }); + + it('does not allow overlapping write transactions', async () => { + const filename = createDbFile(); + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename, + max_pool_size: 2 + }) + ) + ); + + try { + await orm.em.execute('create table write_overlap_test (id integer primary key, value text)', [], 'run'); + await orm.em.execute(`insert into write_overlap_test (id, value) values (1, 'first')`, [], 'run'); + + let releaseFirstWriter!: () => void; + const releaseFirstWriterPromise = new Promise((resolve) => { + releaseFirstWriter = resolve; + }); + let firstWriterReady!: () => void; + const firstWriterReadyPromise = new Promise((resolve) => { + firstWriterReady = resolve; + }); + + const firstWriter = orm.em.fork().transactional(async (transactionalEntityManager) => { + await transactionalEntityManager.execute( + `update write_overlap_test set value = 'held' where id = 1`, + [], + 'run' + ); + firstWriterReady(); + await releaseFirstWriterPromise; + }); + + await firstWriterReadyPromise; + + let secondWriterCompletedBeforeRelease = false; + const secondWriter = orm.em + .fork() + .transactional(async (transactionalEntityManager) => { + await transactionalEntityManager.execute( + `update write_overlap_test set value = 'second' where id = 1`, + [], + 'run' + ); + }) + .then( + () => ({ status: 'fulfilled' as const }), + (error) => ({ status: 'rejected' as const, error }) + ); + + const secondWriterState = await Promise.race([ + secondWriter.then((result) => { + secondWriterCompletedBeforeRelease = result.status == 'fulfilled'; + return result; + }), + new Promise<{ status: 'pending' }>((resolve) => setTimeout(() => resolve({ status: 'pending' }), 50)) + ]); + + expect(secondWriterCompletedBeforeRelease).toBe(false); + const valueBeforeRelease = await orm.em + .fork() + .getKysely<{ write_overlap_test: { id: number; value: string } }>({ type: 'read' }) + .selectFrom('write_overlap_test') + .select('value') + .where('id', '=', 1) + .execute(); + expect(valueBeforeRelease).toEqual([{ value: 'first' }]); + + releaseFirstWriter(); + await firstWriter; + + if (secondWriterState.status == 'pending') { + await expect(secondWriter).resolves.toEqual({ status: 'fulfilled' }); + } else { + expect(secondWriterState.status).toBe('rejected'); + await orm.em.fork().transactional(async (transactionalEntityManager) => { + await transactionalEntityManager.execute( + `update write_overlap_test set value = 'second' where id = 1`, + [], + 'run' + ); + }); + } + + const finalValue = await orm.em.execute<{ value: string }[]>('select value from write_overlap_test where id = 1'); + expect(finalValue).toEqual([{ value: 'second' }]); + } finally { + await orm.close(true); + } + }); + + it('keeps in-memory SQLite storage on a single connection', async () => { + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: ':memory:', + max_pool_size: 3 + }) + ) + ); + + try { + const driver = orm.em.getDriver(); + expect(driver.getConnection('read')).toBe(driver.getConnection('write')); + } finally { + await orm.close(true); + } + }); + + it('runs SQLite migrations through the service migration agent', async () => { + const filename = createDbFile(); + + await using agent = new MikroOrmMigrationAgent({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename + }); + + await agent.run({ + direction: Direction.Up, + migrations: [] + }); + + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename + }) + ) + ); + + try { + const tables = await orm.em + .getConnection() + .execute<{ name: string }[]>(`select name from sqlite_master where type = 'table' order by name`); + const tableNames = tables.map((row) => row.name); + + expect(tableNames).toContain('powersync_migration_locks'); + expect(tableNames).toContain('mikro_orm_migrations'); + expect(tableNames).toContain('instance'); + expect(tableNames).toContain('sync_rules'); + expect(tableNames).toContain('bucket_data'); + expect(tableNames).toContain('write_checkpoints'); + + const indexes = await orm.em + .getConnection() + .execute<{ name: string }[]>(`select name from sqlite_master where type = 'index' order by name`); + const indexNames = indexes.map((row) => row.name); + + expect(indexNames).toContain('bucket_data_bucket_op_index'); + expect(indexNames).toContain('bucket_parameters_lookup_index'); + expect(indexNames).toContain('bucket_parameters_source_index'); + expect(indexNames).toContain('current_data_pending_delete_index'); + expect(indexNames).toContain('source_table_lookup'); + expect(indexNames).toContain('write_checkpoints_user_checkpoint_index'); + expect(indexNames).toContain('write_checkpoints_requested_at_index'); + } finally { + await orm.close(true); + } + }); + + it('configures MySQL migrations outside MikroORM transactions', () => { + const options = createMySqlMikroOrmOptions( + normalizeMikroOrmMySqlStorageConfig({ + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: env.MIKROORM_MYSQL_STORAGE_TEST_URI || 'mysql://repl_user:good_password@localhost:3306/powersync' + }) + ); + + expect(options.migrations?.transactional).toBe(false); + expect(options.migrations?.allOrNothing).toBe(false); + }); + + it.skipIf(!env.MIKROORM_MYSQL_STORAGE_TEST_URI)( + 'runs MySQL migrations through the service migration agent without transactional DDL', + async () => { + const config = { + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: env.MIKROORM_MYSQL_STORAGE_TEST_URI + } as const; + const normalizedConfig = normalizeMikroOrmMySqlStorageConfig(config); + const setupOrm = await MikroORM.init(createMySqlMikroOrmOptions(normalizedConfig)); + + try { + await dropMySqlMigrationTables(setupOrm); + } finally { + await setupOrm.close(true); + } + + await using agent = new MikroOrmMigrationAgent(config); + + await agent.run({ + direction: Direction.Up, + migrations: [] + }); + + const orm = await MikroORM.init(createMySqlMikroOrmOptions(normalizedConfig)); + + try { + const tables = await orm.em.getConnection().execute<{ tableName: string }[]>( + ` + select table_name as tableName + from information_schema.tables + where table_schema = database() + order by table_name + ` + ); + const tableNames = tables.map((row) => row.tableName); + + expect(tableNames).toContain('powersync_mikroorm_migration_locks'); + expect(tableNames).toContain('mikro_orm_migrations'); + expect(tableNames).toContain('instance'); + expect(tableNames).toContain('sync_rules'); + expect(tableNames).toContain('bucket_data'); + expect(tableNames).toContain('write_checkpoints'); + } finally { + await orm.close(true); + } + } + ); + + it('hydrates SQLite-specific storage columns as service-facing values', async () => { + const filename = createDbFile(); + + await using agent = new MikroOrmMigrationAgent({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename + }); + + await agent.run({ + direction: Direction.Up, + migrations: [] + }); + + const orm = await MikroORM.init( + createSqliteMikroOrmOptions( + normalizeMikroOrmSqliteStorageConfig({ + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename + }) + ) + ); + + try { + const em = orm.em.fork(); + const bucketRow = em.create(sqliteMikroOrmStorageDialect.bucketDataEntity, { + id: 'test-bucket-row', + groupId: 1, + bucketName: 'bucket[]', + opId: 42n, + op: 'PUT', + sourceTable: null, + sourceKey: Buffer.from('source-key'), + tableName: null, + rowId: null, + checksum: 99n, + data: null, + targetOp: null + }); + + const syncPlan: storage.SerializedSyncPlan = { + plan: { + version: 1, + dataSources: [], + buckets: [], + parameterIndexes: [], + streams: [] + }, + compatibility: { + edition: 1, + overrides: {} + }, + eventDescriptors: {} + }; + + const syncRulesRow = em.create(sqliteMikroOrmStorageDialect.syncRulesEntity, { + state: storage.SyncRuleState.PROCESSING, + snapshotDone: false, + snapshotLsn: null, + lastCheckpoint: 123n, + lastCheckpointLsn: null, + noCheckpointBefore: null, + slotName: 'test_slot', + lastCheckpointTs: null, + lastKeepaliveTs: null, + lastFatalError: null, + lastFatalErrorTs: null, + keepaliveOp: 124n, + storageVersion: storage.CURRENT_STORAGE_VERSION, + content: 'bucket_definitions: []', + syncPlan + }); + + em.persist([bucketRow, syncRulesRow]); + await em.flush(); + em.clear(); + + const storedBucketRow = await em.findOneOrFail(sqliteMikroOrmStorageDialect.bucketDataEntity, { + id: 'test-bucket-row' + }); + const storedSyncRulesRow = await em.findOneOrFail(sqliteMikroOrmStorageDialect.syncRulesEntity, { + id: syncRulesRow.id + }); + + expect(storedBucketRow.opId).toBe(42n); + expect(storedBucketRow.checksum).toBe(99n); + expect(storedBucketRow.sourceKey).toBeInstanceOf(Buffer); + expect(storedBucketRow.sourceKey?.toString()).toBe('source-key'); + expect(storedSyncRulesRow.lastCheckpoint).toBe(123n); + expect(storedSyncRulesRow.keepaliveOp).toBe(124n); + expect(storedSyncRulesRow.syncPlan).toEqual(syncPlan); + } finally { + await orm.close(true); + } + }); +}); + +async function dropMySqlMigrationTables(orm: MikroORM): Promise { + const connection = orm.em.getConnection(); + + for (const table of [ + 'write_checkpoints', + 'bucket_parameters', + 'current_data', + 'bucket_data', + 'source_tables', + 'sync_rules', + 'instance', + 'mikro_orm_migrations', + 'powersync_mikroorm_migration_locks' + ]) { + await connection.execute(`drop table if exists \`${table}\``, [], 'run'); + } +} diff --git a/modules/module-mikroorm-storage/test/src/mysql-storage.test.ts b/modules/module-mikroorm-storage/test/src/mysql-storage.test.ts new file mode 100644 index 000000000..5d8a59313 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/mysql-storage.test.ts @@ -0,0 +1,46 @@ +import { register } from '@powersync/service-core-tests'; +import { updateSyncRulesFromYaml } from '@powersync/service-core'; +import { describe, expect, it } from 'vitest'; +import type { MikroOrmBucketStorageFactory } from '../../src/index.js'; +import { env } from './env.js'; +import { MIKRO_ORM_MYSQL_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe.skipIf(!env.MIKROORM_MYSQL_STORAGE_TEST_URI).sequential('MikroORM MySQL Sync Bucket Storage', () => { + it('creates the schema and stores sync rules', async () => { + await using factory = (await MIKRO_ORM_MYSQL_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +bucket_definitions: + global: + data: [] +`, + { + validate: false + } + ) + ); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId + } + ]); + }); + + for (let storageVersion of TEST_STORAGE_VERSIONS) { + describe(`Parameters - v${storageVersion}`, () => + register.registerDataStorageParameterTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion })); + + describe(`Data - v${storageVersion}`, () => + register.registerDataStorageDataTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion })); + + describe(`Checkpoints - v${storageVersion}`, () => + register.registerDataStorageCheckpointTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion })); + + describe(`Compaction - v${storageVersion}`, () => { + register.registerCompactTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion }); + register.registerParameterCompactTests({ ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion }); + }); + } +}); diff --git a/modules/module-mikroorm-storage/test/src/setup.ts b/modules/module-mikroorm-storage/test/src/setup.ts new file mode 100644 index 000000000..b14ebcec9 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/setup.ts @@ -0,0 +1,11 @@ +import { container } from '@powersync/lib-services-framework'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; +import { beforeAll, beforeEach } from 'vitest'; + +beforeAll(async () => { + container.registerDefaults(); +}); + +beforeEach(async () => { + METRICS_HELPER.resetMetrics(); +}); diff --git a/modules/module-mikroorm-storage/test/src/storage-provider.test.ts b/modules/module-mikroorm-storage/test/src/storage-provider.test.ts new file mode 100644 index 000000000..8d5453ab2 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/storage-provider.test.ts @@ -0,0 +1,38 @@ +import { storage, system } from '@powersync/service-core'; +import { describe, expect, it } from 'vitest'; +import { MIKRO_ORM_SQLITE_STORAGE_TYPE, MikroOrmStorageProvider } from '../../src/index.js'; + +const RESOLVED_CONFIG = { + storage: { + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: ':memory:' + }, + slot_name_prefix: 'test_' +} as const; + +function getStorageOptions(serviceMode: system.ServiceContextMode): storage.GetStorageOptions { + return { + resolvedConfig: RESOLVED_CONFIG as unknown as storage.GetStorageOptions['resolvedConfig'], + serviceMode + }; +} + +describe('MikroORM SQLite storage provider', () => { + it('rejects split service runners', async () => { + const provider = new MikroOrmStorageProvider(MIKRO_ORM_SQLITE_STORAGE_TYPE); + + await expect(provider.getStorage(getStorageOptions(system.ServiceContextMode.API))).rejects.toThrow( + 'MikroORM SQLite storage only supports the unified service runner' + ); + await expect(provider.getStorage(getStorageOptions(system.ServiceContextMode.SYNC))).rejects.toThrow( + 'MikroORM SQLite storage only supports the unified service runner' + ); + }); + + it('allows the unified service runner', async () => { + const provider = new MikroOrmStorageProvider(MIKRO_ORM_SQLITE_STORAGE_TYPE); + const activeStorage = await provider.getStorage(getStorageOptions(system.ServiceContextMode.UNIFIED)); + + await activeStorage.shutDown(); + }); +}); diff --git a/modules/module-mikroorm-storage/test/src/storage.test.ts b/modules/module-mikroorm-storage/test/src/storage.test.ts new file mode 100644 index 000000000..d9e147c01 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/storage.test.ts @@ -0,0 +1,21 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { MIKRO_ORM_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe('Sync Bucket Validation', register.registerBucketValidationTests); + +for (let storageVersion of TEST_STORAGE_VERSIONS) { + describe(`MikroORM SQLite Sync Bucket Storage - Parameters - v${storageVersion}`, () => + register.registerDataStorageParameterTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`MikroORM SQLite Sync Bucket Storage - Data - v${storageVersion}`, () => + register.registerDataStorageDataTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`MikroORM SQLite Sync Bucket Storage - Checkpoints - v${storageVersion}`, () => + register.registerDataStorageCheckpointTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion })); + + describe(`MikroORM SQLite Sync Bucket Storage - Compaction - v${storageVersion}`, () => { + register.registerCompactTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion }); + register.registerParameterCompactTests({ ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion }); + }); +} diff --git a/modules/module-mikroorm-storage/test/src/storage_bench.test.ts b/modules/module-mikroorm-storage/test/src/storage_bench.test.ts new file mode 100644 index 000000000..e4d91cb6a --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/storage_bench.test.ts @@ -0,0 +1,40 @@ +import type { StorageBenchmarkResult } from '@powersync/service-core-tests'; +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { env } from './env.js'; +import { MIKRO_ORM_MYSQL_STORAGE_FACTORY, MIKRO_ORM_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +const results: StorageBenchmarkResult[] = []; +register.registerStorageBenchmarkSummary(results); + +describe.sequential('MikroORM SQLite Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...MIKRO_ORM_SQLITE_STORAGE_FACTORY, storageVersion }, + { + storageName: 'mikroorm:sqlite', + storageVersion, + results + } + ); + }); + } +}); + +describe + .skipIf(!env.MIKROORM_MYSQL_STORAGE_TEST_URI) + .sequential('MikroORM MySQL Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...MIKRO_ORM_MYSQL_STORAGE_FACTORY, storageVersion }, + { + storageName: 'mikroorm:mysql', + storageVersion, + results + } + ); + }); + } + }); diff --git a/modules/module-mikroorm-storage/test/src/storage_sync.test.ts b/modules/module-mikroorm-storage/test/src/storage_sync.test.ts new file mode 100644 index 000000000..ab6ff1760 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/storage_sync.test.ts @@ -0,0 +1,14 @@ +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { MIKRO_ORM_SQLITE_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +describe('sync - MikroORM SQLite', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`storage v${storageVersion}`, () => { + register.registerSyncTests(MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory, { + storageVersion, + tableIdStrings: MIKRO_ORM_SQLITE_STORAGE_FACTORY.tableIdStrings + }); + }); + } +}); diff --git a/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts b/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts new file mode 100644 index 000000000..820af4af2 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/sync-rules-storage.test.ts @@ -0,0 +1,174 @@ +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { describe, expect, it } from 'vitest'; +import type { MikroOrmBucketStorageFactory } from '../../src/index.js'; +import { MIKRO_ORM_SQLITE_STORAGE_FACTORY } from './util.js'; + +describe('MikroORM SyncRules storage', () => { + const syncRules = updateSyncRulesFromYaml( + ` +bucket_definitions: + mybucket: + data: [] +`, + { + validate: false + } + ); + + it('stores and resolves managed write checkpoints', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + + const first = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '5/0' } }]) + ).writeCheckpoints.get('user1')!; + const second = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '6/0' } }]) + ).writeCheckpoints.get('user1')!; + + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '4/0' } })).resolves.toBeNull(); + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '5/0' } })).resolves.toBe(first); + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1', heads: { '1': '6/0' } })).resolves.toBe(second); + }); + + it('watches checkpoint and managed write checkpoint changes', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + const abortController = new AbortController(); + + try { + const iterator = bucketStorage + .watchCheckpointChanges({ user_id: 'user1', signal: abortController.signal }) + [Symbol.asyncIterator](); + + const writeCheckpoint = ( + await bucketStorage.createManagedWriteCheckpoints([{ user_id: 'user1', heads: { '1': '5/0' } }]) + ).writeCheckpoints.get('user1')!; + + const em = factory.orm.em.fork(); + const row = await em.findOneOrFail(factory.dialect.syncRulesEntity, { + id: stream.replicationStreamId + }); + em.assign(row, { + lastCheckpoint: 0n, + lastCheckpointLsn: '5/0' + }); + await em.flush(); + factory.checkpointWatcher.notify(); + + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { + base: { + checkpoint: 0n, + lsn: '5/0' + }, + writeCheckpoint + } + }); + } finally { + abortController.abort(); + } + }); + + it('resolves custom write checkpoints from the write checkpoint entity', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + const bucketStorage = factory.getInstance(stream); + bucketStorage.setWriteCheckpointMode(storage.WriteCheckpointMode.CUSTOM); + + const em = factory.orm.em.fork(); + const row = em.create(factory.dialect.writeCheckpointEntity, { + id: 'custom-user1', + syncRulesId: stream.replicationStreamId, + userId: 'user1', + checkpoint: 42n, + heads: null, + checkpointRequestedAt: null, + createdAt: new Date() + }); + em.persist(row); + await em.flush(); + + await expect(bucketStorage.lastWriteCheckpoint({ user_id: 'user1' })).resolves.toBe(42n); + }); + + it('marks newly resolved source tables as requiring an initial snapshot', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +bucket_definitions: + global: + data: + - SELECT * FROM lists +`, + { + validate: false + } + ) + ); + const bucketStorage = factory.getInstance(stream); + await using writer = await bucketStorage.createWriter({ + defaultSchema: 'public', + zeroLSN: '0/0', + storeCurrentData: true + }); + + const resolved = await writer.resolveTables({ + connection_id: 1, + source: { + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: 123, + schema: 'public', + name: 'lists', + replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] + }, + idGenerator: () => 'lists-table' + }); + + expect(resolved.tables[0]?.snapshotComplete).toBe(false); + + await writer.markTableSnapshotDone(resolved.tables, '0/1'); + const resolvedAgain = await writer.resolveTables({ + connection_id: 1, + source: { + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: 123, + schema: 'public', + name: 'lists', + replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] + } + }); + + expect(resolvedAgain.tables[0]?.snapshotComplete).toBe(true); + }); + + it('returns active and processing streams as replicating streams', async () => { + await using factory = (await MIKRO_ORM_SQLITE_STORAGE_FACTORY.factory()) as MikroOrmBucketStorageFactory; + const stream = await factory.updateSyncRules(syncRules); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId, + state: storage.SyncRuleState.PROCESSING + } + ]); + + const em = factory.orm.em.fork(); + const row = await em.findOneOrFail(factory.dialect.syncRulesEntity, { + id: stream.replicationStreamId + }); + em.assign(row, { state: storage.SyncRuleState.ACTIVE }); + await em.flush(); + + await expect(factory.getReplicatingReplicationStreams()).resolves.toMatchObject([ + { + replicationStreamId: stream.replicationStreamId, + state: storage.SyncRuleState.ACTIVE + } + ]); + }); +}); diff --git a/modules/module-mikroorm-storage/test/src/util.ts b/modules/module-mikroorm-storage/test/src/util.ts new file mode 100644 index 000000000..105e6c6b8 --- /dev/null +++ b/modules/module-mikroorm-storage/test/src/util.ts @@ -0,0 +1,62 @@ +import { storage, SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core'; +import { + createMySqlMikroOrmStorageFactory, + createSqliteMikroOrmStorageFactory, + MIKRO_ORM_MYSQL_STORAGE_TYPE, + MIKRO_ORM_SQLITE_STORAGE_TYPE, + normalizeMikroOrmMySqlStorageConfig, + normalizeMikroOrmSqliteStorageConfig +} from '../../src/index.js'; +import { env } from './env.js'; + +const BASE_CONFIG = { + type: MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: ':memory:' +} as const; + +export const MIKRO_ORM_SQLITE_STORAGE_FACTORY: storage.TestStorageConfig = { + tableIdStrings: true, + factory: async () => { + const config = normalizeMikroOrmSqliteStorageConfig(BASE_CONFIG); + const factory = await createSqliteMikroOrmStorageFactory({ + config, + slotNamePrefix: 'test_' + }); + await factory.orm.schema.update(); + return factory; + } +}; + +export const MIKRO_ORM_MYSQL_STORAGE_FACTORY: storage.TestStorageConfig = { + tableIdStrings: true, + factory: async () => { + const config = normalizeMikroOrmMySqlStorageConfig({ + type: MIKRO_ORM_MYSQL_STORAGE_TYPE, + uri: env.MIKROORM_MYSQL_STORAGE_TEST_URI + }); + const factory = await createMySqlMikroOrmStorageFactory({ + config, + slotNamePrefix: 'test_' + }); + await dropMySqlStorageTables(factory.orm); + await factory.orm.schema.create(); + return factory; + } +}; + +export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; + +async function dropMySqlStorageTables(orm: Awaited>['orm']) { + const connection = orm.em.getConnection(); + for (const table of [ + 'write_checkpoints', + 'bucket_parameters', + 'current_data', + 'bucket_data', + 'source_tables', + 'sync_rules', + 'instance' + ]) { + await connection.execute(`drop table if exists \`${table}\``, [], 'run'); + } +} diff --git a/modules/module-mikroorm-storage/test/tsconfig.json b/modules/module-mikroorm-storage/test/tsconfig.json new file mode 100644 index 000000000..5bced458d --- /dev/null +++ b/modules/module-mikroorm-storage/test/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../tsconfig.tests.json", + "compilerOptions": { + "declarationDir": "dist/@types", + "tsBuildInfoFile": "dist/.tsbuildinfo", + "lib": ["ES2022", "esnext.disposable"], + "rootDir": "src" + }, + "include": ["src"], + "references": [ + { + "path": "../" + }, + { + "path": "../../../packages/service-core-tests" + } + ] +} diff --git a/modules/module-mikroorm-storage/tsconfig.json b/modules/module-mikroorm-storage/tsconfig.json new file mode 100644 index 000000000..048854242 --- /dev/null +++ b/modules/module-mikroorm-storage/tsconfig.json @@ -0,0 +1,25 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [ + { + "path": "../../packages/types" + }, + { + "path": "../../packages/sync-rules" + }, + { + "path": "../../packages/service-core" + }, + { + "path": "../../libs/lib-services" + } + ] +} diff --git a/modules/module-mikroorm-storage/vitest.config.ts b/modules/module-mikroorm-storage/vitest.config.ts new file mode 100644 index 000000000..285c29477 --- /dev/null +++ b/modules/module-mikroorm-storage/vitest.config.ts @@ -0,0 +1,7 @@ +import { serviceIntegrationTestConfig } from '../test_config'; + +const config = serviceIntegrationTestConfig(__dirname); +config.test ??= {}; +config.test.testTimeout = 30_000; + +export default config; diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts index 895a0729b..de347530b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts @@ -12,6 +12,8 @@ import { SyncConfigDefinition } from './models.js'; +export const BUCKET_DATA_BUCKET_OP_INDEX = 'bucket_op'; + export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo { constructor( upstream: ConstructorParameters[0], diff --git a/modules/module-mongodb-storage/src/utils/test-utils.ts b/modules/module-mongodb-storage/src/utils/test-utils.ts index 922520a54..a90da3a9b 100644 --- a/modules/module-mongodb-storage/src/utils/test-utils.ts +++ b/modules/module-mongodb-storage/src/utils/test-utils.ts @@ -1,5 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { TestStorageOptions } from '@powersync/service-core'; +import { framework, PowerSyncMigrationManager, TestStorageOptions } from '@powersync/service-core'; +import { MongoMigrationAgent } from '../migrations/MongoMigrationAgent.js'; import { MongoBucketStorage, MongoBucketStorageOptions } from '../storage/MongoBucketStorage.js'; import { MongoReportStorage } from '../storage/MongoReportStorage.js'; import { PowerSyncMongo } from '../storage/implementation/db.js'; @@ -7,12 +8,19 @@ import { PowerSyncMongo } from '../storage/implementation/db.js'; export type MongoTestStorageOptions = { url: string; isCI: boolean; + clientOptions?: mongo.MongoClientOptions; + runMigrations?: boolean; + internalOptions?: MongoBucketStorageOptions; } & Omit; export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) { return { factory: async (options?: TestStorageOptions) => { - const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); + if (factoryOptions.runMigrations) { + await runMongoMigrations(factoryOptions.url); + } + + const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.clientOptions); // None of the tests insert data into this collection, so it was never created if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) { @@ -40,7 +48,11 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) { return async (options?: TestStorageOptions) => { - const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); + if (factoryOptions.runMigrations) { + await runMongoMigrations(factoryOptions.url); + } + + const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.clientOptions); await db.createConnectionReportingCollection(); @@ -52,13 +64,24 @@ export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTest }; } -export const connectMongoForTests = (url: string, isCI: boolean) => { +export const connectMongoForTests = (url: string, isCI: boolean, options: mongo.MongoClientOptions = {}) => { // Short timeout for tests, to fail fast when the server is not available. // Slightly longer timeouts for CI, to avoid arbitrary test failures const client = new mongo.MongoClient(url, { connectTimeoutMS: isCI ? 15_000 : 5_000, socketTimeoutMS: isCI ? 15_000 : 5_000, - serverSelectionTimeoutMS: isCI ? 15_000 : 2_500 + serverSelectionTimeoutMS: isCI ? 15_000 : 2_500, + ...options }); return new PowerSyncMongo(client); }; + +async function runMongoMigrations(url: string) { + await using migrationManager: PowerSyncMigrationManager = new framework.migrations.MigrationManager(); + await using migrationAgent = new MongoMigrationAgent({ type: 'mongodb', uri: url }); + + migrationManager.registerMigrationAgent(migrationAgent); + await migrationManager.migrate({ + direction: framework.migrations.Direction.Up + }); +} diff --git a/modules/module-mongodb-storage/test/src/storage.test.ts b/modules/module-mongodb-storage/test/src/storage.test.ts index 5606af296..f1127a26d 100644 --- a/modules/module-mongodb-storage/test/src/storage.test.ts +++ b/modules/module-mongodb-storage/test/src/storage.test.ts @@ -3,6 +3,7 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; +import type { MongoBucketStorage } from '../../src/index.js'; import { env } from './env.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; @@ -235,6 +236,41 @@ bucket_definitions: describe('Sync Bucket Validation', register.registerBucketValidationTests); +test('Mongo v3 bucket data collections use the optimized bucket/op index', async () => { + await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); + const replicationStream = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +bucket_definitions: + global: + data: + - SELECT * FROM test +`, + { storageVersion: 3 } + ) + ); + + const bucketStorage = factory.getInstance(replicationStream); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + void writer; + + const mongoFactory = factory as MongoBucketStorage; + const collections = await mongoFactory.db.listBucketDataCollectionsV3(Number(replicationStream.replicationStreamId)); + expect(collections).toHaveLength(1); + const indexes = await collections[0].indexes(); + expect(indexes).toContainEqual( + expect.objectContaining({ + name: 'bucket_op', + key: { + '_id.b': 1, + '_id.o': 1, + checksum: 1, + op: 1 + } + }) + ); +}); + describe('Mongo Sync Bucket Storage - split operations', () => register.registerDataStorageDataTests( mongoTestStorageFactoryGenerator({ diff --git a/modules/module-mongodb-storage/test/src/storage_bench.test.ts b/modules/module-mongodb-storage/test/src/storage_bench.test.ts new file mode 100644 index 000000000..f62cf7cef --- /dev/null +++ b/modules/module-mongodb-storage/test/src/storage_bench.test.ts @@ -0,0 +1,35 @@ +import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js'; +import type { StorageBenchmarkResult } from '@powersync/service-core-tests'; +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { env } from './env.js'; +import { TEST_STORAGE_VERSIONS } from './util.js'; + +const MONGO_BENCHMARK_STORAGE_FACTORY = mongoTestStorageFactoryGenerator({ + url: env.MONGO_TEST_URL, + isCI: env.CI, + runMigrations: true, + clientOptions: { + connectTimeoutMS: 60_000, + socketTimeoutMS: 1_200_000, + serverSelectionTimeoutMS: 60_000 + } +}); + +const results: StorageBenchmarkResult[] = []; +register.registerStorageBenchmarkSummary(results); + +describe.sequential('MongoDB Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...MONGO_BENCHMARK_STORAGE_FACTORY, storageVersion }, + { + storageName: 'mongodb', + storageVersion, + results + } + ); + }); + } +}); diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 6de7294a8..4232fc471 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -15,6 +15,7 @@ export type PostgresBucketStorageOptions = { config: NormalizedPostgresStorageConfig; replicationStreamNamePrefix: string; checksumCacheTtlMs?: number; + bucketDataQueryHook?: (db: lib_postgres.DatabaseClient, query: pg_wire.Statement) => Promise; }; export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { @@ -57,7 +58,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { db: this.db, replicationStream, batchLimits: this.options.config.batch_limits, - checksumCacheTtlMs: this.options.checksumCacheTtlMs + checksumCacheTtlMs: this.options.checksumCacheTtlMs, + bucketDataQueryHook: this.options.bucketDataQueryHook }); if (!options?.skipLifecycleHooks) { this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage)); diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index fc078dba6..681c905f1 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -47,6 +47,7 @@ export type PostgresSyncRulesStorageOptions = { write_checkpoint_mode?: storage.WriteCheckpointMode; batchLimits: RequiredOperationBatchLimits; checksumCacheTtlMs?: number; + bucketDataQueryHook?: (db: lib_postgres.DatabaseClient, query: Statement) => Promise; }; /** diff --git a/modules/module-postgres-storage/src/utils/test-utils.ts b/modules/module-postgres-storage/src/utils/test-utils.ts index 6f044600a..5fbea6a57 100644 --- a/modules/module-postgres-storage/src/utils/test-utils.ts +++ b/modules/module-postgres-storage/src/utils/test-utils.ts @@ -1,7 +1,10 @@ import { createLogger, logger as defaultLogger, transports } from '@powersync/lib-services-framework'; import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core'; import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js'; -import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js'; +import { + PostgresBucketStorageFactory, + type PostgresBucketStorageOptions +} from '../storage/PostgresBucketStorageFactory.js'; import { PostgresReportStorage } from '../storage/PostgresReportStorage.js'; import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js'; import { truncateTables } from './db.js'; @@ -13,6 +16,7 @@ export type PostgresTestStorageOptions = { * This allows for providing a custom PostgresMigrationAgent. */ migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent; + bucketDataQueryHook?: PostgresBucketStorageOptions['bucketDataQueryHook']; }; export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) { @@ -102,7 +106,8 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) { return new PostgresBucketStorageFactory({ config: TEST_CONNECTION_OPTIONS, - replicationStreamNamePrefix: 'test_' + replicationStreamNamePrefix: 'test_', + bucketDataQueryHook: factoryOptions.bucketDataQueryHook }); } catch (ex) { // Vitest does not display these errors nicely when using the `await using` syntx diff --git a/modules/module-postgres-storage/test/src/storage_bench.test.ts b/modules/module-postgres-storage/test/src/storage_bench.test.ts new file mode 100644 index 000000000..9f68046d2 --- /dev/null +++ b/modules/module-postgres-storage/test/src/storage_bench.test.ts @@ -0,0 +1,29 @@ +import type { StorageBenchmarkResult } from '@powersync/service-core-tests'; +import { register } from '@powersync/service-core-tests'; +import { describe } from 'vitest'; +import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js'; + +const results: StorageBenchmarkResult[] = []; +register.registerStorageBenchmarkSummary(results); + +const scenarios = register.DEFAULT_STORAGE_BENCHMARK_SCENARIOS.filter( + (scenario) => + // scenario.todo_row_count < 1_000_000 && + scenario.max_bucket_count == null || scenario.max_bucket_count <= 1_000 +); + +describe.sequential('Postgres Sync Bucket Storage Benchmarks', () => { + for (const storageVersion of TEST_STORAGE_VERSIONS) { + describe.skipIf(storageVersion !== 1)(`v${storageVersion}`, () => { + register.registerStorageBenchmarks( + { ...POSTGRES_STORAGE_FACTORY, storageVersion }, + { + storageName: 'postgresql', + storageVersion, + scenarios, + results + } + ); + }); + } +}); diff --git a/modules/module-postgres-storage/test/src/util.ts b/modules/module-postgres-storage/test/src/util.ts index 105214d7f..c28b5bb20 100644 --- a/modules/module-postgres-storage/test/src/util.ts +++ b/modules/module-postgres-storage/test/src/util.ts @@ -1,4 +1,6 @@ -import { SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core'; +import type { DatabaseClient } from '@powersync/lib-service-postgres'; +import { LEGACY_STORAGE_VERSION } from '@powersync/service-core'; +import type { Statement } from '@powersync/service-jpgwire'; import path from 'path'; import { fileURLToPath } from 'url'; import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js'; @@ -18,6 +20,29 @@ const BASE_CONFIG = { export const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG); +let explainedBucketDataQuery = false; +let bucketDataQueryCount = 0; + +async function explainBucketDataQuery(db: DatabaseClient, query: Statement) { + bucketDataQueryCount++; + const explainQuery = Number(process.env.POWERSYNC_STORAGE_BENCHMARK_EXPLAIN_QUERY ?? 1); + if ( + process.env.POWERSYNC_STORAGE_BENCHMARK_EXPLAIN !== 'true' || + explainedBucketDataQuery || + bucketDataQueryCount != explainQuery + ) { + return; + } + explainedBucketDataQuery = true; + + const rows = await db.queryRows>({ + statement: `EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT TEXT) ${query.statement}`, + params: query.params + }); + const plan = rows.map((row) => Object.values(row)[0]).join('\n'); + console.log(`\nPostgres bucket data query plan\n${plan}\n`); +} + /** * Vitest tries to load the migrations via .ts files which fails. * For tests this links to the relevant .js files correctly @@ -30,10 +55,11 @@ class TestPostgresMigrationAgent extends PostgresMigrationAgent { export const POSTGRES_STORAGE_SETUP = postgresTestSetup({ url: env.PG_STORAGE_TEST_URL, - migrationAgent: (config) => new TestPostgresMigrationAgent(config) + migrationAgent: (config) => new TestPostgresMigrationAgent(config), + bucketDataQueryHook: explainBucketDataQuery }); export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP; export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory; -export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; +export const TEST_STORAGE_VERSIONS = [LEGACY_STORAGE_VERSION]; diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index e83a686ca..2e86d778a 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -36,6 +36,8 @@ "devDependencies": { "@powersync/lib-service-postgres": "workspace:*", "@powersync/service-core-tests": "workspace:*", + "@powersync/service-module-mikroorm-storage": "workspace:*", + "@powersync/service-module-drizzle-storage": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-postgres-storage": "workspace:*", "@types/semver": "^7.5.4" diff --git a/modules/module-postgres/test/src/env.ts b/modules/module-postgres/test/src/env.ts index ef3a1e09a..77b62cbd5 100644 --- a/modules/module-postgres/test/src/env.ts +++ b/modules/module-postgres/test/src/env.ts @@ -1,11 +1,17 @@ import { utils } from '@powersync/lib-services-framework'; export const env = utils.collectEnvironmentVariables({ - PG_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_test'), - PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_storage_test'), + PG_TEST_URL: utils.type.string.default('postgres://postgres:mypassword@localhost:5432/powersync_test'), + PG_STORAGE_TEST_URL: utils.type.string.default( + 'postgres://postgres:mypassword@localhost:5432/powersync_storage_test' + ), MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'), CI: utils.type.boolean.default('false'), SLOW_TESTS: utils.type.boolean.default('false'), - TEST_MONGO_STORAGE: utils.type.boolean.default('true'), - TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') + TEST_MONGO_STORAGE: utils.type.boolean.default('false'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('false'), + TEST_MIKROORM_SQLITE_STORAGE: utils.type.boolean.default('true'), + MIKROORM_SQLITE_STORAGE_TEST_FILENAME: utils.type.string.default(''), + TEST_DRIZZLE_SQLITE_STORAGE: utils.type.boolean.default('false'), + DRIZZLE_SQLITE_STORAGE_TEST_FILENAME: utils.type.string.default('') }); diff --git a/modules/module-postgres/test/src/util.ts b/modules/module-postgres/test/src/util.ts index 401153100..6e1bfc25a 100644 --- a/modules/module-postgres/test/src/util.ts +++ b/modules/module-postgres/test/src/util.ts @@ -10,8 +10,14 @@ import { TestStorageFactory } from '@powersync/service-core'; import * as pgwire from '@powersync/service-jpgwire'; +import * as drizzle_storage from '@powersync/service-module-drizzle-storage'; +import * as mikroorm_storage from '@powersync/service-module-mikroorm-storage'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as postgres_storage from '@powersync/service-module-postgres-storage'; +import { existsSync } from 'node:fs'; +import { unlink } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, TestOptions } from 'vitest'; import { env } from './env.js'; @@ -26,6 +32,59 @@ export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils. url: env.PG_STORAGE_TEST_URL }); +const DEFAULT_MIKROORM_SQLITE_STORAGE_FILENAME = join( + tmpdir(), + `powersync-postgres-replication-mikroorm-storage-${process.pid}-${process.env.VITEST_WORKER_ID ?? '0'}.sqlite` +); + +const MIKROORM_SQLITE_STORAGE_FILENAME = + env.MIKROORM_SQLITE_STORAGE_TEST_FILENAME || DEFAULT_MIKROORM_SQLITE_STORAGE_FILENAME; + +const DEFAULT_DRIZZLE_SQLITE_STORAGE_FILENAME = join( + tmpdir(), + `powersync-postgres-replication-drizzle-storage-${process.pid}-${process.env.VITEST_WORKER_ID ?? '0'}.sqlite` +); +const DRIZZLE_SQLITE_STORAGE_FILENAME = + env.DRIZZLE_SQLITE_STORAGE_TEST_FILENAME || DEFAULT_DRIZZLE_SQLITE_STORAGE_FILENAME; + +export const INITIALIZED_MIKROORM_SQLITE_STORAGE_FACTORY: TestStorageConfig = { + tableIdStrings: true, + factory: async (options) => { + if (!options?.doNotClear && existsSync(MIKROORM_SQLITE_STORAGE_FILENAME)) { + await unlink(MIKROORM_SQLITE_STORAGE_FILENAME); + } + + const config = mikroorm_storage.normalizeMikroOrmSqliteStorageConfig({ + type: mikroorm_storage.MIKRO_ORM_SQLITE_STORAGE_TYPE, + filename: MIKROORM_SQLITE_STORAGE_FILENAME + }); + const factory = await mikroorm_storage.createSqliteMikroOrmStorageFactory({ + config, + slotNamePrefix: 'test_' + }); + await factory.orm.schema.update(); + return factory; + } +}; + +export const INITIALIZED_DRIZZLE_SQLITE_STORAGE_FACTORY: TestStorageConfig = { + tableIdStrings: true, + factory: async (options) => { + if (!options?.doNotClear && existsSync(DRIZZLE_SQLITE_STORAGE_FILENAME)) { + await unlink(DRIZZLE_SQLITE_STORAGE_FILENAME); + } + const factory = drizzle_storage.createSqliteDrizzleStorageFactory({ + config: drizzle_storage.normalizeDrizzleSqliteStorageConfig({ + type: drizzle_storage.DRIZZLE_SQLITE_STORAGE_TYPE, + filename: DRIZZLE_SQLITE_STORAGE_FILENAME + }), + slotNamePrefix: 'test_' + }); + drizzle_storage.runSqliteDrizzleMigrations(factory.runtime); + return factory; + } +}; + const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; export interface StorageVersionTestContext { @@ -58,6 +117,14 @@ export function describeWithStorage( if (env.TEST_POSTGRES_STORAGE) { describeFactory('postgres', INITIALIZED_POSTGRES_STORAGE_FACTORY); } + + if (env.TEST_MIKROORM_SQLITE_STORAGE) { + describeFactory('mikroorm sqlite', INITIALIZED_MIKROORM_SQLITE_STORAGE_FACTORY); + } + + if (env.TEST_DRIZZLE_SQLITE_STORAGE) { + describeFactory('drizzle sqlite', INITIALIZED_DRIZZLE_SQLITE_STORAGE_FACTORY); + } } export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({ diff --git a/modules/module-postgres/test/tsconfig.json b/modules/module-postgres/test/tsconfig.json index b69636818..df3b2c86e 100644 --- a/modules/module-postgres/test/tsconfig.json +++ b/modules/module-postgres/test/tsconfig.json @@ -19,6 +19,12 @@ { "path": "../../module-mongodb-storage" }, + { + "path": "../../module-mikroorm-storage" + }, + { + "path": "../../module-drizzle-storage" + }, { "path": "../../module-postgres-storage" } diff --git a/package.json b/package.json index 7c26758d9..31e4dd25f 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "author": "PowerSync", "scripts": { "check-updates": "ncu -u --deep", + "benchmark:storage": "node scripts/compare-storage-benchmarks.mts", "validate:tsconfig-references": "node scripts/validate-tsconfig-references.mts", "format": "prettier --write .", "format:check": "prettier --check .", diff --git a/packages/schema/package.json b/packages/schema/package.json index c32bc0778..2deb505d0 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -26,6 +26,8 @@ "@powersync/service-module-postgres-storage": "workspace:*", "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-mikroorm-storage": "workspace:*", + "@powersync/service-module-drizzle-storage": "workspace:*", "@powersync/service-module-convex": "workspace:*", "@powersync/service-module-mysql": "workspace:*", "@powersync/service-module-mssql": "workspace:*", diff --git a/packages/schema/src/scripts/compile-json-schema.ts b/packages/schema/src/scripts/compile-json-schema.ts index 9d68b053b..5e70a7481 100644 --- a/packages/schema/src/scripts/compile-json-schema.ts +++ b/packages/schema/src/scripts/compile-json-schema.ts @@ -1,4 +1,6 @@ import { ConvexConnectionConfig } from '@powersync/service-module-convex/types'; +import { DrizzleStorageConfig } from '@powersync/service-module-drizzle-storage/types'; +import { MikroOrmStorageConfig } from '@powersync/service-module-mikroorm-storage/types'; import { MongoStorageConfig } from '@powersync/service-module-mongodb-storage/types'; import { MongoConnectionConfig } from '@powersync/service-module-mongodb/types'; import { MSSQLConnectionConfig } from '@powersync/service-module-mssql/types'; @@ -25,7 +27,10 @@ const mergedDataSourceConfig = configFile.genericDataSourceConfig .or(PostgresConnectionConfig) .or(ConvexConnectionConfig); -const mergedStorageConfig = configFile.GenericStorageConfig.or(PostgresStorageConfig).or(MongoStorageConfig); +const mergedStorageConfig = configFile.GenericStorageConfig.or(PostgresStorageConfig) + .or(MongoStorageConfig) + .or(MikroOrmStorageConfig) + .or(DrizzleStorageConfig); const mergedConfig = t.object({ ...baseShape, diff --git a/packages/schema/tsconfig.json b/packages/schema/tsconfig.json index 0ac12addf..fcf6723fe 100644 --- a/packages/schema/tsconfig.json +++ b/packages/schema/tsconfig.json @@ -21,6 +21,9 @@ { "path": "../../modules/module-mongodb-storage" }, + { + "path": "../../modules/module-drizzle-storage" + }, { "path": "../../modules/module-mysql" }, diff --git a/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts b/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts new file mode 100644 index 000000000..f585cc7ac --- /dev/null +++ b/packages/service-core-tests/src/benchmarks/register-storage-benchmarks.ts @@ -0,0 +1,507 @@ +import { createCoreAPIMetrics, JwtPayload, storage, sync, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { afterAll, test } from 'vitest'; +import { + BATCH_OPTIONS, + deploySyncRules, + METRICS_HELPER, + PARSE_OPTIONS, + resolveTestTable +} from '../test-utils/test-utils-index.js'; + +const BENCHMARK_SYNC_RULES = ` +bucket_definitions: + by_list: + parameters: + - SELECT id AS list_id FROM lists + data: + - SELECT id, list_id, owner_id, description, completed FROM todos WHERE list_id = bucket.list_id + by_user: + parameters: + - SELECT owner_id AS owner_id FROM lists + data: + - SELECT id, list_id, owner_id, description, completed FROM todos WHERE owner_id = bucket.owner_id +`; + +export interface StorageBenchmarkScenario { + name: string; + /** Number of todo rows to persist and expose through data buckets. */ + todo_row_count: number; + /** Number of list rows to persist and expose as parameter buckets. */ + list_row_count: number; + /** Number of distinct users spread over list and todo rows. */ + user_count: number; + /** Flush pending source rows after this many saves. */ + flush_every: number; + /** Maximum expected number of resolved buckets for this scenario. */ + max_bucket_count?: number; + /** Optional per-scenario Vitest timeout. */ + timeout_ms?: number; +} + +export interface RegisterStorageBenchmarkOptions { + storageName: string; + storageVersion?: number; + scenarios?: StorageBenchmarkScenario[]; + results?: StorageBenchmarkResult[]; + timeout_ms?: number; + progress_interval_ms?: number; +} + +export interface StorageBenchmarkResult { + storage: string; + version: number | null; + scenario: string; + source_rows: number; + buckets: number; + write_ms: number; + write_rows_per_second: number; + write_mebibytes_per_second: number; + sync_drain_ms: number; + ops: number; + read_mebibytes_per_second: number; +} + +export interface StorageBenchmarkOutput { + schema_version: 1; + generated_at: string; + results: StorageBenchmarkResult[]; +} + +export const STORAGE_BENCHMARK_OUTPUT_PATH_ENV = 'POWERSYNC_STORAGE_BENCHMARK_OUTPUT'; + +const STORAGE_BENCHMARK_ROW_PERMUTATIONS = [ + { + name: '1k-todos', + todo_row_count: 1_000, + timeout_ms: 300_000 + }, + { + name: '10k-todos', + todo_row_count: 10_000, + timeout_ms: 600_000 + }, + { + name: '100k-todos', + todo_row_count: 100_000, + timeout_ms: 1_800_000 + }, + { + name: '1m-todos', + todo_row_count: 1_000_000, + timeout_ms: 10_800_000 + } +]; + +const STORAGE_BENCHMARK_BUCKET_PERMUTATIONS = [ + { name: '200-buckets', list_row_count: 100, bucket_count: 200, minimum_timeout_ms: 0 }, + { name: '1k-buckets', list_row_count: 500, bucket_count: 1_000, minimum_timeout_ms: 0 }, + { name: '10k-buckets', list_row_count: 5_000, bucket_count: 10_000, minimum_timeout_ms: 900_000 }, + { name: '20k-buckets', list_row_count: 10_000, bucket_count: 20_000, minimum_timeout_ms: 1_200_000 } +]; + +export const DEFAULT_STORAGE_BENCHMARK_SCENARIOS: StorageBenchmarkScenario[] = + STORAGE_BENCHMARK_ROW_PERMUTATIONS.flatMap((rowPermutation) => + STORAGE_BENCHMARK_BUCKET_PERMUTATIONS.map((bucketPermutation) => ({ + name: `${rowPermutation.name}-${bucketPermutation.name}`, + todo_row_count: rowPermutation.todo_row_count, + list_row_count: bucketPermutation.list_row_count, + user_count: bucketPermutation.list_row_count, + flush_every: 1_000, + max_bucket_count: bucketPermutation.bucket_count, + timeout_ms: Math.max(rowPermutation.timeout_ms, bucketPermutation.minimum_timeout_ms) + })) + ); + +export function registerStorageBenchmarks( + configOrFactory: storage.TestStorageConfig | storage.TestStorageFactory, + options: RegisterStorageBenchmarkOptions +) { + const config: storage.TestStorageConfig = + typeof configOrFactory == 'function' ? { factory: configOrFactory, tableIdStrings: true } : configOrFactory; + const storageVersion = options.storageVersion ?? config.storageVersion; + const scenarios = options.scenarios ?? DEFAULT_STORAGE_BENCHMARK_SCENARIOS; + + createCoreAPIMetrics(METRICS_HELPER.metricsEngine); + + for (const scenario of scenarios) { + test( + `storage benchmark - ${scenario.name}`, + async () => { + const result = await runStorageBenchmark( + config, + { + ...options, + storageVersion + }, + scenario + ); + + options.results?.push(result); + console.log(formatStorageBenchmarkResultLine(result)); + }, + scenario.timeout_ms ?? options.timeout_ms ?? 1_200_000 + ); + } +} + +export function registerStorageBenchmarkSummary( + results: StorageBenchmarkResult[], + title = 'Storage benchmark results' +) { + afterAll(async () => { + if (results.length == 0) { + return; + } + + console.log(`\n${title}\n${formatStorageBenchmarkResults(results)}`); + + const outputPath = process.env[STORAGE_BENCHMARK_OUTPUT_PATH_ENV]; + if (outputPath != null && outputPath.length > 0) { + await writeStorageBenchmarkResults(outputPath, results); + console.log(`\nWrote storage benchmark results to ${outputPath}`); + } + }); +} + +export async function writeStorageBenchmarkResults(outputPath: string, results: StorageBenchmarkResult[]) { + const output: StorageBenchmarkOutput = { + schema_version: 1, + generated_at: new Date().toISOString(), + results + }; + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(output, null, 2)}\n`, 'utf8'); +} + +export function formatStorageBenchmarkResults(results: StorageBenchmarkResult[]) { + const rows = [ + '| Storage | Version | Scenario | Source Rows | Buckets | Write ms | Write rows/s | Write MiB/s | Sync drain ms | Ops | Read MiB/s |', + '| --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |' + ]; + + for (const result of results) { + rows.push( + `| ${result.storage} | ${result.version ?? ''} | ${result.scenario} | ${result.source_rows} | ${ + result.buckets + } | ${formatNumber(result.write_ms)} | ${formatNumber(result.write_rows_per_second)} | ${formatNumber( + result.write_mebibytes_per_second + )} | ${formatNumber(result.sync_drain_ms)} | ${result.ops} | ${formatNumber(result.read_mebibytes_per_second)} |` + ); + } + + return rows.join('\n'); +} + +async function runStorageBenchmark( + config: storage.TestStorageConfig, + options: Required> & + Omit, + scenario: StorageBenchmarkScenario +): Promise { + validateScenario(scenario); + + await using factory = await config.factory(); + const syncRules = await deploySyncRules( + factory, + updateSyncRulesFromYaml(BENCHMARK_SYNC_RULES, { + storageVersion: options.storageVersion + }) + ); + + const bucketStorage = factory.getInstance(syncRules.stream); + const label = benchmarkLabel(options.storageName, options.storageVersion, scenario.name); + const sourceRows = scenario.todo_row_count + scenario.list_row_count; + const sourceBytes = calculateSourceBytes(scenario); + + const writeStarted = performance.now(); + await persistBenchmarkRows({ + config, + bucketStorage, + scenario, + label, + progressIntervalMs: options.progress_interval_ms ?? 5_000 + }); + const writeMs = performance.now() - writeStarted; + + const drain = await drainSyncStream({ + bucketStorage, + label, + scenario, + progressIntervalMs: options.progress_interval_ms ?? 5_000 + }); + + return { + storage: options.storageName, + version: options.storageVersion ?? null, + scenario: scenario.name, + source_rows: sourceRows, + buckets: drain.buckets, + write_ms: writeMs, + write_rows_per_second: sourceRows / (writeMs / 1_000), + write_mebibytes_per_second: sourceBytes / 1024 / 1024 / (writeMs / 1_000), + sync_drain_ms: drain.elapsedMs, + ops: drain.ops, + read_mebibytes_per_second: drain.bytes / 1024 / 1024 / (drain.elapsedMs / 1_000) + }; +} + +async function persistBenchmarkRows(options: { + config: storage.TestStorageConfig; + bucketStorage: storage.SyncRulesBucketStorage; + scenario: StorageBenchmarkScenario; + label: string; + progressIntervalMs: number; +}) { + const { config, bucketStorage, scenario, label } = options; + const sourceRows = scenario.todo_row_count + scenario.list_row_count; + const started = performance.now(); + let lastProgress = started; + let savedRows = 0; + let pendingRows = 0; + + await using writer = await bucketStorage.createWriter(BATCH_OPTIONS); + const listsTable = await resolveTestTable(writer, 'lists', ['id'], config, 1); + const todosTable = await resolveTestTable(writer, 'todos', ['id'], config, 2); + + await writer.markAllSnapshotDone('0/1'); + + async function saveRow(row: storage.SaveOptions) { + await writer.save(row); + savedRows++; + pendingRows++; + + if (pendingRows >= scenario.flush_every) { + await writer.flush(); + pendingRows = 0; + maybeLogProgress(); + } + } + + function maybeLogProgress(force = false) { + const now = performance.now(); + if (force || now - lastProgress >= options.progressIntervalMs) { + lastProgress = now; + console.log( + `[${label}] write progress source_rows=${savedRows}/${sourceRows} elapsed_ms=${formatNumber(now - started)}` + ); + } + } + + for (let i = 0; i < scenario.list_row_count; i++) { + const listId = listRowId(i); + await saveRow({ + sourceTable: listsTable, + tag: storage.SaveOperationTag.INSERT, + after: listRow(i, scenario), + afterReplicaId: listId + }); + } + + for (let i = 0; i < scenario.todo_row_count; i++) { + const todoId = todoRowId(i); + await saveRow({ + sourceTable: todosTable, + tag: storage.SaveOperationTag.INSERT, + after: todoRow(i, scenario), + afterReplicaId: todoId + }); + } + + maybeLogProgress(true); + await writer.commit('1/1'); +} + +async function drainSyncStream(options: { + bucketStorage: storage.SyncRulesBucketStorage; + label: string; + scenario: StorageBenchmarkScenario; + progressIntervalMs: number; +}) { + const { bucketStorage, label, scenario, progressIntervalMs } = options; + const started = performance.now(); + const tracker = new sync.RequestTracker(METRICS_HELPER.metricsEngine); + const controller = new AbortController(); + const maxBucketCount = configuredMaxBucketCount(scenario); + const syncContext = new sync.SyncContext({ + maxBuckets: maxBucketCount + 10, + maxParameterQueryResults: maxBucketCount + 10, + maxDataFetchConcurrency: 2 + }); + + let lines = 0; + let ops = 0; + let bytes = 0; + let buckets = 0; + let completed = false; + + const heartbeat = setInterval(() => { + console.log( + `[${label}] sync stream heartbeat lines=${lines} ops=${ops} bytes=${bytes} elapsed_ms=${formatNumber( + performance.now() - started + )}` + ); + }, progressIntervalMs); + heartbeat.unref?.(); + + try { + const stream = sync.streamResponse({ + syncContext, + bucketStorage, + syncRules: bucketStorage.getParsedSyncRules(PARSE_OPTIONS), + params: { + buckets: [], + include_checksum: true, + raw_data: true + }, + tracker, + token: new JwtPayload({ sub: 'user_00000000', exp: Date.now() / 1_000 + 3_600 }), + isEncodingAsBson: false, + signal: controller.signal + }); + + for await (const rawLine of stream) { + if (rawLine == null) { + continue; + } + + lines++; + const line = parseSyncLine(rawLine); + bytes += Buffer.byteLength(typeof rawLine == 'string' ? rawLine : JSONBig.stringify(rawLine)); + + if (line != null && 'checkpoint' in line) { + buckets = line.checkpoint.buckets.length; + } else if (line != null && 'checkpoint_diff' in line) { + buckets = Math.max(buckets, line.checkpoint_diff.updated_buckets.length); + } else if (line != null && 'data' in line) { + ops += line.data.data.length; + } else if (line != null && 'checkpoint_complete' in line) { + completed = true; + break; + } + } + } finally { + controller.abort(); + clearInterval(heartbeat); + } + + if (!completed) { + throw new Error(`[${label}] sync stream ended before checkpoint_complete`); + } + + return { + buckets, + ops, + bytes, + elapsedMs: performance.now() - started + }; +} + +function parseSyncLine(line: string | object) { + if (typeof line != 'string') { + return line as any; + } + + try { + return JSONBig.parse(line) as any; + } catch { + return null; + } +} + +function validateScenario(scenario: StorageBenchmarkScenario) { + if (scenario.todo_row_count < 1) { + throw new Error(`Benchmark scenario ${scenario.name} must have at least one todo row`); + } + if (scenario.list_row_count < 1) { + throw new Error(`Benchmark scenario ${scenario.name} must have at least one list row`); + } + if (scenario.user_count < 1) { + throw new Error(`Benchmark scenario ${scenario.name} must have at least one user`); + } + if (scenario.flush_every < 1) { + throw new Error(`Benchmark scenario ${scenario.name} must flush at least every one row`); + } + + const maxBucketCount = configuredMaxBucketCount(scenario); + const buckets = expectedBucketCount(scenario); + if (buckets > maxBucketCount) { + throw new Error( + `Benchmark scenario ${scenario.name} creates ${buckets} buckets, exceeding max_bucket_count=${maxBucketCount}` + ); + } +} + +function expectedBucketCount(scenario: StorageBenchmarkScenario) { + return scenario.list_row_count + Math.min(scenario.list_row_count, scenario.user_count); +} + +function configuredMaxBucketCount(scenario: StorageBenchmarkScenario) { + return scenario.max_bucket_count ?? 1_000; +} + +function calculateSourceBytes(scenario: StorageBenchmarkScenario) { + let bytes = 0; + for (let i = 0; i < scenario.list_row_count; i++) { + bytes += Buffer.byteLength(JSONBig.stringify(listRow(i, scenario))); + } + for (let i = 0; i < scenario.todo_row_count; i++) { + bytes += Buffer.byteLength(JSONBig.stringify(todoRow(i, scenario))); + } + return bytes; +} + +function listRow(index: number, scenario: StorageBenchmarkScenario) { + return { + id: listRowId(index), + owner_id: userId(index % scenario.user_count), + name: `List ${index}` + }; +} + +function todoRow(index: number, scenario: StorageBenchmarkScenario) { + const listIndex = index % scenario.list_row_count; + return { + id: todoRowId(index), + list_id: listRowId(listIndex), + owner_id: userId(listIndex % scenario.user_count), + description: `Todo ${index} for ${listRowId(listIndex)}`, + completed: index % 3 == 0 ? 1 : 0 + }; +} + +function benchmarkLabel(storageName: string, storageVersion: number | undefined, scenarioName: string) { + return `${storageName}/v${storageVersion ?? 'default'}/${scenarioName}`; +} + +function listRowId(index: number) { + return `list_${String(index).padStart(8, '0')}`; +} + +function todoRowId(index: number) { + return `todo_${String(index).padStart(8, '0')}`; +} + +function userId(index: number) { + return `user_${String(index).padStart(8, '0')}`; +} + +function formatStorageBenchmarkResultLine(result: StorageBenchmarkResult) { + return `[${result.storage}/v${result.version ?? 'default'}/${result.scenario}] result write_ms=${formatNumber( + result.write_ms + )} write_rows_per_second=${formatNumber(result.write_rows_per_second)} write_mebibytes_per_second=${formatNumber( + result.write_mebibytes_per_second + )} sync_drain_ms=${formatNumber(result.sync_drain_ms)} ops=${result.ops} buckets=${ + result.buckets + } read_mebibytes_per_second=${formatNumber(result.read_mebibytes_per_second)}`; +} + +function formatNumber(value: number) { + if (Number.isInteger(value)) { + return String(value); + } + return value.toFixed(value >= 100 ? 0 : 2); +} diff --git a/packages/service-core-tests/src/tests/tests-index.ts b/packages/service-core-tests/src/tests/tests-index.ts index a40468a32..2b1e28aca 100644 --- a/packages/service-core-tests/src/tests/tests-index.ts +++ b/packages/service-core-tests/src/tests/tests-index.ts @@ -1,3 +1,4 @@ +export * from '../benchmarks/register-storage-benchmarks.js'; export * from './register-bucket-validation-tests.js'; export * from './register-compacting-tests.js'; export * from './register-data-storage-checkpoint-tests.js'; diff --git a/packages/service-core/src/storage/StorageEngine.ts b/packages/service-core/src/storage/StorageEngine.ts index a53819ccb..b7ddfec4d 100644 --- a/packages/service-core/src/storage/StorageEngine.ts +++ b/packages/service-core/src/storage/StorageEngine.ts @@ -5,6 +5,7 @@ import { ActiveStorage, StorageProvider } from './StorageProvider.js'; export type StorageEngineOptions = { configuration: ResolvedPowerSyncConfig; + serviceMode: string; }; export interface StorageEngineListener { @@ -45,7 +46,8 @@ export class StorageEngine extends BaseObserver { logger.info('Starting Storage Engine...'); const { configuration } = this.options; this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type)!.getStorage({ - resolvedConfig: configuration + resolvedConfig: configuration, + serviceMode: this.options.serviceMode }); this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage)); this.currentActiveStorage.onFatalError?.((error) => { diff --git a/packages/service-core/src/storage/StorageProvider.ts b/packages/service-core/src/storage/StorageProvider.ts index f2404bdf0..ed1ddad7d 100644 --- a/packages/service-core/src/storage/StorageProvider.ts +++ b/packages/service-core/src/storage/StorageProvider.ts @@ -19,6 +19,7 @@ export interface ActiveStorage { export interface GetStorageOptions { // TODO: This should just be the storage config. Update once the slot name prefix coupling has been removed from the storage resolvedConfig: util.ResolvedPowerSyncConfig; + serviceMode: string; } /** diff --git a/packages/service-core/src/system/ServiceContext.ts b/packages/service-core/src/system/ServiceContext.ts index a45166ff7..02684164c 100644 --- a/packages/service-core/src/system/ServiceContext.ts +++ b/packages/service-core/src/system/ServiceContext.ts @@ -62,7 +62,8 @@ export class ServiceContextContainer implements ServiceContext { this.lifeCycleEngine = new LifeCycledSystem(); this.storageEngine = new storage.StorageEngine({ - configuration + configuration, + serviceMode: this.serviceMode }); this.storageEngine.registerListener({ storageFatalError: (error) => { diff --git a/plans/mikroorm-bucket-storage.md b/plans/mikroorm-bucket-storage.md new file mode 100644 index 000000000..617dbe81d --- /dev/null +++ b/plans/mikroorm-bucket-storage.md @@ -0,0 +1,379 @@ +# MikroORM Bucket Storage Module Plan + +## Goal + +Build an experimental bucket storage module backed by MikroORM, with SQLite as the first supported driver and a clean path for future SQL drivers. The module must share entity definitions and storage algorithms by default, while isolating database-specific behavior behind small dialect and migration-lock abstractions. + +The first public storage identifier is: + +```yaml +storage: + type: mikroorm:sqlite +``` + +SQLite storage is intended for single-process unified PowerSync service deployments only. Future drivers can add cross-process notifications and split-runner support through the dialect layer. + +## Architecture Decisions + +- Start from the PowerSync bucket storage contract, not from a database-first schema design. +- Use MikroORM v7 and its SQL driver stack. +- Use common `defineEntity` schema constants and common TypeScript entity classes for all entities that do not actually vary by driver. +- Do not create empty driver-specific entity subclasses. Add subclasses only when a driver needs methods, hooks, or different behavior. +- Use MikroORM built-in property types for the first SQLite implementation. Avoid custom bigint/blob/json builders unless a later driver proves they are necessary. +- Keep storage algorithms common. Add dialect methods only for query streaming, notification behavior, lock behavior, or raw SQL that cannot be expressed cleanly through typed ORM calls. +- Split high-volume writes into persisted chunks so one `BucketBatch` does not keep every operation in one MikroORM unit of work. +- Stream hot read paths and compaction inputs. Do not load full bucket or compaction result sets into memory. +- Run the shared core storage/sync suites and selected module-postgres replication suites against the MikroORM SQLite storage. + +## Module Layout + +```text +modules/module-mikroorm-storage/ + package.json + tsconfig.json + vitest.config.ts + README.md + src/ + index.ts + mikro-orm.config.ts + module/ + MikroOrmStorageModule.ts + types/ + types.ts + entities/ + entities-index.ts + common/ + bucket-data.schema.ts + bucket-parameters.schema.ts + current-data.schema.ts + instance.schema.ts + source-table.schema.ts + sync-rules.schema.ts + write-checkpoint.schema.ts + drivers/ + sqlite/ + SqliteMikroOrmStorageFactory.ts + SqliteMigrationLockManager.ts + sqlite-config.ts + sqlite-dialect.ts + migrations/ + AbstractMikroOrmMigrationLockManager.ts + MikroOrmMigrationAgent.ts + NoOpMigrationStore.ts + sqlite/ + Migration*_InitialSqliteStorage.ts + storage/ + MikroOrmBucketBatch.ts + MikroOrmPersistedBatch.ts + MikroOrmBucketStorageFactory.ts + MikroOrmCompactor.ts + MikroOrmPersistedReplicationStream.ts + MikroOrmReportStorage.ts + MikroOrmStorageDialect.ts + MikroOrmStorageProvider.ts + MikroOrmSyncRulesStorage.ts + storage-index.ts + unsupported.ts + test/ + src/ + migrations.test.ts + storage.test.ts + storage-provider.test.ts + storage_sync.test.ts + sync-rules-storage.test.ts + util.ts +``` + +## Entity Model + +Use common entity classes with MikroORM `defineEntity` schema constants: + +```ts +export const BucketDataSchema = defineEntity({ + name: 'BucketData', + tableName: 'bucket_data', + properties: { + id: p.string().primary(), + groupId: p.integer().fieldName('group_id'), + bucketName: p.string().fieldName('bucket_name'), + opId: p.bigint('bigint') + } +}); + +export class BucketData extends BucketDataSchema.class {} +BucketDataSchema.setClass(BucketData); +``` + +The common entity set is: + +- `bucket_data` +- `bucket_parameters` +- `current_data` +- `instance` +- `source_tables` +- `sync_rules` +- `write_checkpoints` + +Indexes belong in the common entity schema when they represent logical access patterns shared by drivers. SQLite must include indexes for bucket reads, parameter lookup reads, source-table resolution, current-data lookups, write checkpoints, and sync-rule state queries. + +Source-table metadata must be JSON-safe. Normalize replica id column type ids to `number` before storing them so MikroORM JSON serialization never sees `bigint` values. + +Future driver subclasses should follow the documented MikroORM pattern only when the subclass adds real value: + +```ts +export class PostgresBucketData extends BucketDataSchema.class { + // driver-specific methods or hooks +} +BucketDataSchema.setClass(PostgresBucketData); +``` + +## Dialect Interface + +Common storage code depends on `MikroOrmStorageDialect`, not SQLite imports. + +```ts +export interface MikroOrmStorageDialect { + readonly type: string; + readonly entityClasses: EntityClass[]; + readonly bucketDataEntity: EntityClass; + readonly bucketParametersEntity: EntityClass; + readonly currentDataEntity: EntityClass; + readonly instanceEntity: EntityClass; + readonly sourceTableEntity: EntityClass; + readonly syncRulesEntity: EntityClass; + readonly writeCheckpointEntity: EntityClass; + streamBucketDataRows(options: MikroOrmBucketDataStreamOptions): AsyncIterable; + createCheckpointWatcher(): MikroOrmCheckpointWatcher; +} +``` + +The dialect owns: + +- public storage identifier, for example `mikroorm:sqlite` +- entity class registration for MikroORM and migration tooling +- streamed bucket-data reads +- checkpoint watch/notify implementation +- future database-specific raw SQL helpers + +SQLite bucket-data streaming should use MikroORM query builder streaming. It must also yield to the event loop before the query so tight single-process polling loops cannot starve replication/checkpoint work. + +## SQLite Configuration + +SQLite config should be small and explicit: + +```yaml +storage: + type: mikroorm:sqlite + filename: ./powersync-storage.sqlite +``` + +Rules: + +- `filename` controls the SQLite database file. SQLite creates the file if it does not exist. +- `:memory:` is supported for focused unit tests, but file-backed databases should be used for integration tests that reopen storage with `doNotClear`. +- Do not expose a migrations path in public config. The module owns its bundled migration path. +- Register the module with the service so `mikroorm:sqlite` can be used in normal self-hosted config. +- Include the module in the service Docker build. +- Ensure native `better-sqlite3` bindings are built in the production Docker image. + +SQLite must reject split API/sync runner modes. It can run in unified service mode and command/tooling contexts. + +## Migrations + +Use MikroORM migrations internally and expose them through the standard PowerSync migration surface. + +Implementation requirements: + +- `MikroOrmMigrationAgent` extends the service migration agent and overrides `run()`. +- The service migration surface triggers the agent. +- MikroORM owns migration discovery and migration state. +- `NoOpMigrationStore` prevents duplicate PowerSync migration bookkeeping. +- `AbstractMikroOrmMigrationLockManager` defines the DB-backed lock contract. +- `SqliteMigrationLockManager` bootstraps its lock table with raw SQLite. + +The raw SQLite lock bootstrap is required. Migration locking has a classic chicken-and-egg problem: migrations normally create tables, but a distributed-safe migration trigger needs a table-backed lock before migrations run. For SQLite, use `CREATE TABLE IF NOT EXISTS` in the lock manager before invoking MikroORM migrations. + +Generate SQLite migration scripts with the MikroORM CLI. Do not hand-write the initial schema migration except for intentional lock-manager bootstrap SQL. + +Migration tests must verify that representative storage tables and indexes are created. + +## Storage Implementation + +Implement the common storage classes in this order: + +1. `MikroOrmStorageProvider` +2. `MikroOrmBucketStorageFactory` +3. `MikroOrmPersistedReplicationStream` +4. `MikroOrmSyncRulesStorage` +5. `MikroOrmBucketBatch` +6. `MikroOrmPersistedBatch` +7. `MikroOrmCompactor` +8. `MikroOrmReportStorage` + +`MikroOrmBucketBatch` owns: + +- listener notifications +- `resolveTables` +- `save` +- `flush` +- `truncate` +- `drop` +- checkpoint commits +- snapshot state +- write checkpoints + +`MikroOrmPersistedBatch` owns: + +- per-transaction `current_data` preload +- bucket data evaluation +- parameter data evaluation +- `bucket_data` inserts +- `bucket_parameters` inserts +- `current_data` upserts and pending-delete markers + +Flush pending operations in bounded chunks, currently `2_000` source operations per persisted transaction. Log successful flushes with source-operation counts and resulting storage-op ranges. + +## Source Tables and Schema Changes + +`resolveTables()` must detect both same-name changes and relation-id changes: + +- Match the active source table by connection, schema, table name, relation id, and normalized replica id columns. +- When a table is renamed, return old source-table rows in `dropTables` if they share the same `relation_id.object_id`. +- When a table changes replica identity or relevant column type metadata, return old source-table rows in `dropTables`. +- New source-table rows must start with `snapshotDone: false` so initial and triggered snapshots are explicit. + +This behavior is required for table recreate, rename, replica identity, and publication-change replication tests. + +## Large Rows and Current Data + +Current-data persistence must tolerate rows that cannot be serialized to BSON or exceed the maximum persisted row size. + +Rules: + +- Use a `15 MiB` current-data row limit, matching existing storage behavior. +- If full-row BSON serialization fails or exceeds the limit, log a warning and store a BSON row where each field value is `undefined`. +- Evaluate sync rules against the truncated row so future TOAST-style updates can still be marked unavailable rather than crashing replication. +- Suppress evaluated bucket rows produced only from a truncated row with no usable object id. This prevents placeholder blank-id bucket ops and follow-up blank-id removes. +- Keep legitimate empty-string ids valid when `data.id` is explicitly present. + +## Reads and Checkpoints + +`MikroOrmSyncRulesStorage.getBucketDataBatch()` should consume dialect-streamed rows and chunk output without buffering the full query result. + +Bucket read output must include subkeys whenever `source_table` and `source_key` are present. + +`MikroOrmBucketStorageFactory.getReplicatingReplicationStreams()` must return both processing and active streams. Active streams still replicate after initial snapshot completion and must remain visible to replication management. + +SQLite checkpoint watching is process-local: + +- writes should notify the in-process watcher +- reads should remain cooperative with the event loop +- split-service deployments must be rejected at provider startup + +## Compaction + +Compaction should stream candidate rows and avoid loading full result sets into memory. + +Use typed ORM calls where they are readable and efficient. Use raw SQL for database-specific operations that are awkward or inefficient through MikroORM, and isolate that SQL behind common methods or dialect helpers. + +Validate compaction through the shared storage sync tests, especially checkpoint invalidation and bucket batch cases. + +## Tests + +The module must import shared PowerSync storage tests instead of relying only on bespoke SQLite tests. + +Required module-level coverage: + +- migration agent tests +- storage provider tests, including SQLite unified-mode guard +- shared storage tests +- shared sync tests across storage protocol versions +- sync-rules storage tests +- compaction tests + +The module-postgres replication tests should be runnable against MikroORM SQLite storage with: + +```sh +TEST_MONGO_STORAGE=false +TEST_POSTGRES_STORAGE=false +TEST_MIKROORM_SQLITE_STORAGE=true +``` + +module-postgres integration tests should use a file-backed SQLite storage filename by default. Allow `MIKROORM_SQLITE_STORAGE_TEST_FILENAME` to override it, but do not default those tests to `:memory:` because resume and `doNotClear` flows need storage state to survive factory reopen. + +Sequential module-postgres files to run against MikroORM SQLite storage: + +- `checkpoints.test.ts` +- `chunked_snapshots.test.ts` +- `large_batch.test.ts` +- `pg_test.test.ts` +- `replica_identity_full.test.ts` +- `replication_retry.test.ts` +- `resuming_snapshots.test.ts` +- `route_api_adapter.test.ts` +- `schema_changes.test.ts` +- `slow_tests.test.ts` +- `storage_combination.test.ts` +- `types/registry.test.ts` +- `validation.test.ts` +- `wal_budget_api.test.ts` +- `wal_budget.test.ts` +- `wal_stream.test.ts` + +Some files are skipped unless their suite-specific environment flags are enabled. + +## Verification Commands + +```sh +source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage build +source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage build:tests +source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-mikroorm-storage test --run +source ~/.nvm/nvm.sh && nvm use && corepack pnpm --filter @powersync/service-module-postgres build:tests +source ~/.nvm/nvm.sh && nvm use && TEST_MONGO_STORAGE=false TEST_POSTGRES_STORAGE=false TEST_MIKROORM_SQLITE_STORAGE=true corepack pnpm --filter @powersync/service-module-postgres test test/src/wal_stream.test.ts --run +source ~/.nvm/nvm.sh && nvm use && TEST_MONGO_STORAGE=false TEST_POSTGRES_STORAGE=false TEST_MIKROORM_SQLITE_STORAGE=true corepack pnpm --filter @powersync/service-module-postgres test test/src/schema_changes.test.ts --run +``` + +Expected baseline: + +- MikroORM storage source build passes. +- MikroORM storage test build passes. +- Full MikroORM storage suite passes with the shared storage/sync tests. +- module-postgres `wal_stream.test.ts` passes against MikroORM SQLite storage. +- module-postgres `schema_changes.test.ts` passes against MikroORM SQLite storage. + +## Build Order From Scratch + +1. Create the package, TypeScript config, Vitest config, exports, README, and workspace wiring. +2. Add common MikroORM entity schemas and classes. +3. Add SQLite config and dialect with entity class registration. +4. Configure MikroORM migrations and generate the SQLite initial migration with the MikroORM CLI. +5. Add the DB-backed migration lock and migration agent override. +6. Add provider config for `mikroorm:sqlite`. +7. Register the module with the service module loader and Docker build. +8. Implement factory and sync-rule persistence. +9. Implement writer creation and `MikroOrmBucketBatch`. +10. Split persisted operation writes into `MikroOrmPersistedBatch`. +11. Implement oversized current-data handling. +12. Implement source-table rename and replica-identity drop detection. +13. Implement streamed bucket reads through the dialect. +14. Implement write checkpoints and checkpoint watching. +15. Implement compaction with streamed reads. +16. Add the SQLite unified-runner guard. +17. Import shared storage and sync tests. +18. Add module-postgres MikroORM SQLite storage test wiring. +19. Run the verification commands and the sequential module-postgres files. + +## Future Driver Guidance + +For Postgres or another SQL driver: + +- reuse common entity definitions first +- add concrete entity subclasses only for real driver-specific behavior +- add a new dialect object with its own entity class list +- implement cross-process checkpoint notification in `createCheckpointWatcher()` +- move raw SQL into dialect methods when query shape differs by database +- generate migrations with the MikroORM CLI for that driver +- add a DB-backed migration lock before invoking the migrator +- run the shared storage/sync suites and relevant replication-module integration tests against the new driver + +This keeps the module abstract without over-abstracting the entity model before another driver proves what varies. diff --git a/plans/sync-bucket-storage-benchmarks.md b/plans/sync-bucket-storage-benchmarks.md new file mode 100644 index 000000000..5e7870c92 --- /dev/null +++ b/plans/sync-bucket-storage-benchmarks.md @@ -0,0 +1,147 @@ +# Sync Bucket Storage Benchmark Suite Plan + +## Goal + +Add a reusable benchmark suite for comparing PowerSync sync bucket storage implementations. The suite should measure the two storage paths that matter most for sync throughput: + +- Persisting a synthetic incoming dataset that touches both parameter lookups and bucket data writes. +- Draining all bucket data through the real sync stream API until `checkpoint_complete`. + +The benchmark harness belongs in `packages/service-core-tests` so every storage module can register the same scenarios without duplicating data generation or stream-drain logic. + +## Benchmark Model + +Use a source-DB-free synthetic dataset written through the storage writer API: + +- `lists` rows are parameter rows. +- `todos` rows are data rows. +- One bucket definition groups todos by list. +- One bucket definition groups todos by user. + +This makes each todo row appear in two bucket families while list rows drive the dynamic parameter hot path. + +Scenario fields are explicit test parameters, not environment variables: + +```ts +interface StorageBenchmarkScenario { + name: string; + todo_row_count: number; + list_row_count: number; + user_count: number; + flush_every: number; + max_bucket_count?: number; + timeout_ms?: number; +} +``` + +Default scenarios are the cross-product of these dimensions: + +| Todo Rows | Timeout | +| --------: | ------: | +| 1,000 | 5 min | +| 10,000 | 10 min | +| 100,000 | 30 min | +| 1,000,000 | 3 h | + +| Resolved Buckets | List Rows | Users | Minimum Timeout | +| ---------------: | --------: | -----: | --------------: | +| 200 | 100 | 100 | - | +| 1,000 | 500 | 500 | - | +| 10,000 | 5,000 | 5,000 | 15 min | +| 20,000 | 10,000 | 10,000 | 20 min | + +This produces 16 scenarios. Every row-count permutation includes the original 200- and 1,000-bucket shapes plus the +new 10,000- and 20,000-bucket shapes. Each scenario passes its own `max_bucket_count` into the sync context for both +bucket and parameter-result limits. All registered scenarios run both the write phase and sync stream drain phase. + +## Harness Requirements + +- Export `registerStorageBenchmarks()` from `@powersync/service-core-tests`. +- Export a benchmark summary printer that emits a markdown table after the suite. +- Accept a normal `TestStorageConfig` or `TestStorageFactory`. +- Accept storage name, storage version, scenario list, timeout, progress interval, and an optional shared result array. +- Write rows with `createWriter()` and flush every `flush_every` source rows. +- Commit once after all rows are written so the drain measures a single checkpoint. +- Drain via `sync.streamResponse()` with `raw_data: true` and stop only when `checkpoint_complete` is received. +- Do not benchmark a separate direct `getBucketDataBatch()` drain; the sync stream drain is the public path under test. +- Emit heartbeat progress logs during long sync drains showing lines, ops, bytes, and elapsed milliseconds. +- Emit write progress logs during long writes. + +## Storage Module Registration + +Each storage module should add a `test/src/storage_bench.test.ts` file and register all supported storage versions: + +- `modules/module-postgres-storage` +- `modules/module-mongodb-storage` +- `modules/module-mikroorm-storage` for SQLite and MySQL + +Storage setup must include migrations or schema setup before benchmark timing so indexes are present. Postgres and MikroORM use their existing test factory migration/schema paths. MongoDB benchmark factories should explicitly run migrations and use longer client socket timeouts so large drains are not killed by the normal fast-fail test timeout settings. + +## Sample Commands + +Run a single storage benchmark file with Vitest's `--run` mode: + +```sh +pnpm --filter @powersync/service-module-postgres-storage test test/src/storage_bench.test.ts --run +``` + +```sh +pnpm --filter @powersync/service-module-mongodb-storage test test/src/storage_bench.test.ts --run +``` + +```sh +pnpm --filter @powersync/service-module-mikroorm-storage test test/src/storage_bench.test.ts --run -t "MikroORM SQLite" +``` + +```sh +MIKROORM_MYSQL_STORAGE_TEST_URI="mysql://repl_user:good_password@localhost:3306/powersync" \ + corepack pnpm --filter @powersync/service-module-mikroorm-storage test test/src/storage_bench.test.ts --run -t "MikroORM MySQL" +``` + +The storage modules use their normal test database environment variables when supplied: + +- `PG_STORAGE_TEST_URL` for Postgres storage benchmarks. +- `MONGO_TEST_URL` for MongoDB storage benchmarks. +- `MIKROORM_MYSQL_STORAGE_TEST_URI` for MikroORM MySQL storage benchmarks. + +## Output + +The summary table should include: + +- Storage +- Version +- Scenario +- Source Rows +- Buckets +- Write ms +- Write rows/s +- Write MiB/s, based on the logical JSON size of source rows +- Sync drain ms +- Ops +- Read MiB/s, based on bytes emitted by the sync stream + +The output is intentionally plain markdown so benchmark results can be pasted into issues, PRs, or follow-up analysis notes. + +## Comparing Storage Implementations + +Use the root comparison CLI to select and run multiple storage implementations: + +```sh +pnpm benchmark:storage +``` + +Missing storage selections are prompted for interactively. They can also be supplied directly: + +```sh +pnpm benchmark:storage --storage drizzle-sqlite,mikroorm-sqlite +``` + +Use `--output ` to retain the combined JSON results. The CLI runs each selected module in a separate Vitest +process, collects its machine-readable output, and prints throughput relative to the fastest selected implementation +for each scenario. + +The CLI also writes a self-contained grouped SVG chart to `storage-benchmark-comparison.svg`, comparing write and read +MiB/s for every storage/version/scenario result. Use `--chart ` to select a different destination. + +A benchmark test can also write its results directly by setting `POWERSYNC_STORAGE_BENCHMARK_OUTPUT` to the desired +JSON path. Without that environment variable, benchmark tests keep their existing console-only behavior. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1a491078e..421f58454 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -94,10 +94,10 @@ importers: version: 6.0.3 vite: specifier: ^8.0.9 - version: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) + version: 8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) ws: specifier: ^8.2.3 version: 8.18.0 @@ -194,7 +194,7 @@ importers: version: 4.17.6 vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) modules/module-convex: dependencies: @@ -263,6 +263,98 @@ importers: specifier: ^5.8.5 version: 5.8.5 + modules/module-drizzle-storage: + dependencies: + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../../libs/lib-services + '@powersync/service-core': + specifier: workspace:* + version: link:../../packages/service-core + '@powersync/service-jsonbig': + specifier: workspace:* + version: link:../../packages/jsonbig + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + better-sqlite3: + specifier: ^12.10.0 + version: 12.10.0 + drizzle-orm: + specifier: ^0.45.2 + version: 0.45.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.15.4)(better-sqlite3@12.10.0)(kysely@0.29.2)(mysql2@3.22.4(@types/node@25.5.0)) + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + uuid: + specifier: 'catalog:' + version: 14.0.0 + devDependencies: + '@powersync/service-core-tests': + specifier: workspace:* + version: link:../../packages/service-core-tests + '@types/better-sqlite3': + specifier: ^7.6.13 + version: 7.6.13 + drizzle-kit: + specifier: ^0.31.10 + version: 0.31.10 + typescript: + specifier: 'catalog:' + version: 6.0.3 + + modules/module-mikroorm-storage: + dependencies: + '@mikro-orm/core': + specifier: ^7.1.4 + version: 7.1.4 + '@mikro-orm/migrations': + specifier: ^7.1.4 + version: 7.1.4(@mikro-orm/core@7.1.4) + '@mikro-orm/mysql': + specifier: ^7.1.4 + version: 7.1.4(@mikro-orm/core@7.1.4)(@types/node@25.5.0) + '@mikro-orm/sql': + specifier: ^7.1.4 + version: 7.1.4(@mikro-orm/core@7.1.4) + '@mikro-orm/sqlite': + specifier: ^7.1.4 + version: 7.1.4(@mikro-orm/core@7.1.4) + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../../libs/lib-services + '@powersync/service-core': + specifier: workspace:* + version: link:../../packages/service-core + '@powersync/service-jsonbig': + specifier: workspace:* + version: link:../../packages/jsonbig + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + uuid: + specifier: 'catalog:' + version: 14.0.0 + devDependencies: + '@mikro-orm/cli': + specifier: ^7.1.4 + version: 7.1.4 + '@powersync/service-core-tests': + specifier: workspace:* + version: link:../../packages/service-core-tests + typescript: + specifier: 'catalog:' + version: 6.0.3 + modules/module-mongodb: dependencies: '@powersync/lib-service-mongodb': @@ -501,6 +593,12 @@ importers: '@powersync/service-core-tests': specifier: workspace:* version: link:../../packages/service-core-tests + '@powersync/service-module-drizzle-storage': + specifier: workspace:* + version: link:../module-drizzle-storage + '@powersync/service-module-mikroorm-storage': + specifier: workspace:* + version: link:../module-mikroorm-storage '@powersync/service-module-mongodb-storage': specifier: workspace:* version: link:../module-mongodb-storage @@ -571,7 +669,7 @@ importers: devDependencies: vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) packages/jsonbig: dependencies: @@ -612,6 +710,12 @@ importers: '@powersync/service-module-convex': specifier: workspace:* version: link:../../modules/module-convex + '@powersync/service-module-drizzle-storage': + specifier: workspace:* + version: link:../../modules/module-drizzle-storage + '@powersync/service-module-mikroorm-storage': + specifier: workspace:* + version: link:../../modules/module-mikroorm-storage '@powersync/service-module-mongodb': specifier: workspace:* version: link:../../modules/module-mongodb @@ -765,7 +869,7 @@ importers: version: link:../sync-rules vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) devDependencies: '@opentelemetry/sdk-metrics': specifier: ^1.30.1 @@ -808,7 +912,7 @@ importers: version: 1.0.0 vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) packages/types: dependencies: @@ -836,6 +940,12 @@ importers: '@powersync/service-module-core': specifier: workspace:* version: link:../modules/module-core + '@powersync/service-module-drizzle-storage': + specifier: workspace:* + version: link:../modules/module-drizzle-storage + '@powersync/service-module-mikroorm-storage': + specifier: workspace:* + version: link:../modules/module-mikroorm-storage '@powersync/service-module-mongodb': specifier: workspace:* version: link:../modules/module-mongodb @@ -1201,6 +1311,9 @@ packages: '@dabh/diagnostics@2.0.3': resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} + '@drizzle-team/brocli@0.10.2': + resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} + '@emnapi/core@1.8.1': resolution: {integrity: sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==} @@ -1219,162 +1332,614 @@ packages: '@emnapi/wasi-threads@1.2.1': resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} + '@esbuild-kit/core-utils@3.3.2': + resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} + deprecated: 'Merged into tsx: https://tsx.hirok.io' + + '@esbuild-kit/esm-loader@2.6.5': + resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} + deprecated: 'Merged into tsx: https://tsx.hirok.io' + + '@esbuild/aix-ppc64@0.25.12': + resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + '@esbuild/aix-ppc64@0.27.0': resolution: {integrity: sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.18.20': + resolution: {integrity: sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.25.12': + resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + '@esbuild/android-arm64@0.27.0': resolution: {integrity: sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==} engines: {node: '>=18'} cpu: [arm64] os: [android] + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.18.20': + resolution: {integrity: sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.25.12': + resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + '@esbuild/android-arm@0.27.0': resolution: {integrity: sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==} engines: {node: '>=18'} cpu: [arm] os: [android] + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.18.20': + resolution: {integrity: sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.25.12': + resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + '@esbuild/android-x64@0.27.0': resolution: {integrity: sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==} engines: {node: '>=18'} cpu: [x64] os: [android] + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.18.20': + resolution: {integrity: sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.25.12': + resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + '@esbuild/darwin-arm64@0.27.0': resolution: {integrity: sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.18.20': + resolution: {integrity: sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.12': + resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + '@esbuild/darwin-x64@0.27.0': resolution: {integrity: sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==} engines: {node: '>=18'} cpu: [x64] os: [darwin] + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.18.20': + resolution: {integrity: sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.25.12': + resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + '@esbuild/freebsd-arm64@0.27.0': resolution: {integrity: sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.18.20': + resolution: {integrity: sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.12': + resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + '@esbuild/freebsd-x64@0.27.0': resolution: {integrity: sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.18.20': + resolution: {integrity: sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.25.12': + resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + '@esbuild/linux-arm64@0.27.0': resolution: {integrity: sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==} engines: {node: '>=18'} cpu: [arm64] os: [linux] + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.18.20': + resolution: {integrity: sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.25.12': + resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + '@esbuild/linux-arm@0.27.0': resolution: {integrity: sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==} engines: {node: '>=18'} cpu: [arm] os: [linux] + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.18.20': + resolution: {integrity: sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.25.12': + resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + '@esbuild/linux-ia32@0.27.0': resolution: {integrity: sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==} engines: {node: '>=18'} cpu: [ia32] os: [linux] + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.18.20': + resolution: {integrity: sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.25.12': + resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + '@esbuild/linux-loong64@0.27.0': resolution: {integrity: sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==} engines: {node: '>=18'} cpu: [loong64] os: [linux] + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.18.20': + resolution: {integrity: sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.25.12': + resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + '@esbuild/linux-mips64el@0.27.0': resolution: {integrity: sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.18.20': + resolution: {integrity: sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.25.12': + resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + '@esbuild/linux-ppc64@0.27.0': resolution: {integrity: sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.18.20': + resolution: {integrity: sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.12': + resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + '@esbuild/linux-riscv64@0.27.0': resolution: {integrity: sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.18.20': + resolution: {integrity: sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.25.12': + resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + '@esbuild/linux-s390x@0.27.0': resolution: {integrity: sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==} engines: {node: '>=18'} cpu: [s390x] os: [linux] + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.18.20': + resolution: {integrity: sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.25.12': + resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + '@esbuild/linux-x64@0.27.0': resolution: {integrity: sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==} engines: {node: '>=18'} cpu: [x64] os: [linux] + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.12': + resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + '@esbuild/netbsd-arm64@0.27.0': resolution: {integrity: sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.18.20': + resolution: {integrity: sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.12': + resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + '@esbuild/netbsd-x64@0.27.0': resolution: {integrity: sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.12': + resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + '@esbuild/openbsd-arm64@0.27.0': resolution: {integrity: sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.18.20': + resolution: {integrity: sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.12': + resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + '@esbuild/openbsd-x64@0.27.0': resolution: {integrity: sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.25.12': + resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + '@esbuild/openharmony-arm64@0.27.0': resolution: {integrity: sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.18.20': + resolution: {integrity: sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.25.12': + resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + '@esbuild/sunos-x64@0.27.0': resolution: {integrity: sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==} engines: {node: '>=18'} cpu: [x64] os: [sunos] + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.18.20': + resolution: {integrity: sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.25.12': + resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + '@esbuild/win32-arm64@0.27.0': resolution: {integrity: sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==} engines: {node: '>=18'} cpu: [arm64] os: [win32] + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.18.20': + resolution: {integrity: sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.25.12': + resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + '@esbuild/win32-ia32@0.27.0': resolution: {integrity: sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==} engines: {node: '>=18'} cpu: [ia32] os: [win32] + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.18.20': + resolution: {integrity: sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.25.12': + resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@esbuild/win32-x64@0.27.0': resolution: {integrity: sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==} engines: {node: '>=18'} cpu: [x64] os: [win32] + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@fastify/ajv-compiler@4.0.5': resolution: {integrity: sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==} @@ -1453,6 +2018,44 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} + '@mikro-orm/cli@7.1.4': + resolution: {integrity: sha512-o1AQgwVJ3i8fGpR+48bSTum/Ar5u38WfyX1HykoPR03M5IdOHJ2QdiwA0o58TsA5xd0U+IlhlNLJEwZ8XCNXGA==} + engines: {node: '>= 22.17.0'} + hasBin: true + + '@mikro-orm/core@7.1.4': + resolution: {integrity: sha512-JPpOoBw2r98hVmsa7kFdt9HDqTa6/x9dhLL2MZRJQt1HwNg7S7A6Eh+nJ21dmB5QfXhu2PXqt03Y+xZZDnUD0w==} + engines: {node: '>= 22.17.0'} + peerDependencies: + dataloader: 2.2.3 + peerDependenciesMeta: + dataloader: + optional: true + + '@mikro-orm/migrations@7.1.4': + resolution: {integrity: sha512-M9rsHpuuaWavoXb0yitgwb+Dy2wzIEzFh5cdJ1NHjPVwyWrTiTTcKmZUiytvHInzLvusuTomRKNRiNUENqnFvQ==} + engines: {node: '>= 22.17.0'} + peerDependencies: + '@mikro-orm/core': 7.1.4 + + '@mikro-orm/mysql@7.1.4': + resolution: {integrity: sha512-Gck977Bp91KD9gB3TO0Trd46jiNw0fSM53kPOiX9WiBNoYdKIli6Yz8mAzYnd63dYb2q67jif5/gjZ2IMYlJIQ==} + engines: {node: '>= 22.17.0'} + peerDependencies: + '@mikro-orm/core': 7.1.4 + + '@mikro-orm/sql@7.1.4': + resolution: {integrity: sha512-YOie3z0Y24XlzWPUrATGJja5U6Ij81gji9yEDT6JkZxJH9uF1tbmZ2qX9WHectnMB5h8JHCic5AMvaflhBviNg==} + engines: {node: '>= 22.17.0'} + peerDependencies: + '@mikro-orm/core': 7.1.4 + + '@mikro-orm/sqlite@7.1.4': + resolution: {integrity: sha512-0Buc7GMkib22SliR4dj4O3Yzs26k8lzRn5xRRpgkLpt+Mlse9ac3Mk+2ilLTwsjHm2yoIIRQbfNnhVk1z0UoSw==} + engines: {node: '>= 22.17.0'} + peerDependencies: + '@mikro-orm/core': 7.1.4 + '@mongodb-js/saslprep@1.3.1': resolution: {integrity: sha512-6nZrq5kfAz0POWyhljnbWQQJQ5uT8oE2ddX303q1uY0tWsivWKgBDXBBvuFPwOqRRalXJuVO9EjOdVtuhLX0zg==} @@ -2195,6 +2798,9 @@ packages: '@types/async@3.2.24': resolution: {integrity: sha512-8iHVLHsCCOBKjCF2KwFe0p9Z3rfM9mL+sSP8btyR5vTjJRAqpBYD28/ZLgXPf0pjG1VxOvtCV/BgXkQbpSe8Hw==} + '@types/better-sqlite3@7.6.13': + resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} + '@types/body-parser@1.19.6': resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} @@ -2476,6 +3082,10 @@ packages: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} + better-sqlite3@12.10.0: + resolution: {integrity: sha512-CyzaZRQKyHkB2ZInfTTl2nvT33EbDpjkLEbE8/Zck3Ll6O0qqvuGdrJ45HgtH+HykRg88ITY3AdreBGN70aBSQ==} + engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x || 26.x} + big-integer@1.6.51: resolution: {integrity: sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==} engines: {node: '>=0.6'} @@ -2491,6 +3101,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -2525,6 +3138,9 @@ packages: buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} @@ -2751,6 +3367,102 @@ packages: resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} engines: {node: '>=12'} + drizzle-kit@0.31.10: + resolution: {integrity: sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw==} + hasBin: true + + drizzle-orm@0.45.2: + resolution: {integrity: sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q==} + peerDependencies: + '@aws-sdk/client-rds-data': '>=3' + '@cloudflare/workers-types': '>=4' + '@electric-sql/pglite': '>=0.2.0' + '@libsql/client': '>=0.10.0' + '@libsql/client-wasm': '>=0.10.0' + '@neondatabase/serverless': '>=0.10.0' + '@op-engineering/op-sqlite': '>=2' + '@opentelemetry/api': ^1.4.1 + '@planetscale/database': '>=1.13' + '@prisma/client': '*' + '@tidbcloud/serverless': '*' + '@types/better-sqlite3': '*' + '@types/pg': '*' + '@types/sql.js': '*' + '@upstash/redis': '>=1.34.7' + '@vercel/postgres': '>=0.8.0' + '@xata.io/client': '*' + better-sqlite3: '>=7' + bun-types: '*' + expo-sqlite: '>=14.0.0' + gel: '>=2' + knex: '*' + kysely: '*' + mysql2: '>=2' + pg: '>=8' + postgres: '>=3' + prisma: '*' + sql.js: '>=1' + sqlite3: '>=5' + peerDependenciesMeta: + '@aws-sdk/client-rds-data': + optional: true + '@cloudflare/workers-types': + optional: true + '@electric-sql/pglite': + optional: true + '@libsql/client': + optional: true + '@libsql/client-wasm': + optional: true + '@neondatabase/serverless': + optional: true + '@op-engineering/op-sqlite': + optional: true + '@opentelemetry/api': + optional: true + '@planetscale/database': + optional: true + '@prisma/client': + optional: true + '@tidbcloud/serverless': + optional: true + '@types/better-sqlite3': + optional: true + '@types/pg': + optional: true + '@types/sql.js': + optional: true + '@upstash/redis': + optional: true + '@vercel/postgres': + optional: true + '@xata.io/client': + optional: true + better-sqlite3: + optional: true + bun-types: + optional: true + expo-sqlite: + optional: true + gel: + optional: true + knex: + optional: true + kysely: + optional: true + mysql2: + optional: true + pg: + optional: true + postgres: + optional: true + prisma: + optional: true + sql.js: + optional: true + sqlite3: + optional: true + ebnf@1.9.1: resolution: {integrity: sha512-uW2UKSsuty9ANJ3YByIQE4ANkD8nqUPO7r6Fwcc1ADKPe9FRdcPpMl3VEput4JSvKBJ4J86npIC2MLP0pYkCuw==} hasBin: true @@ -2774,11 +3486,26 @@ packages: es-module-lexer@2.0.0: resolution: {integrity: sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==} + esbuild@0.18.20: + resolution: {integrity: sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==} + engines: {node: '>=12'} + hasBin: true + + esbuild@0.25.12: + resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} + engines: {node: '>=18'} + hasBin: true + esbuild@0.27.0: resolution: {integrity: sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==} engines: {node: '>=18'} hasBin: true + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + escalade@3.1.2: resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} engines: {node: '>=6'} @@ -2882,6 +3609,9 @@ packages: resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} engines: {node: '>=18'} + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -2945,6 +3675,9 @@ packages: resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} engines: {node: '>=18'} + get-tsconfig@4.14.0: + resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} + github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} @@ -3207,6 +3940,10 @@ packages: kuler@2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} + kysely@0.29.2: + resolution: {integrity: sha512-s6WVJyEZrbm6jhBpiKHsGHyePMrVQKJ85wZCFCr9W4QHv6WTjWIrdvTmO9hDEA3bNK0xkrE2DqrHsXMLWuZpQg==} + engines: {node: '>=22.0.0'} + leven@3.1.0: resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} engines: {node: '>=6'} @@ -3334,6 +4071,9 @@ packages: long@5.2.3: resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + lossless-json@2.0.11: resolution: {integrity: sha512-BP0vn+NGYvzDielvBZaFain/wgeJ1hTvURCqtKvhr1SCPePdaaTanmmcplrHfEJSJOUql7hk4FHwToNJjWRY3g==} @@ -3348,6 +4088,10 @@ packages: resolution: {integrity: sha512-FbAj6lXil6t8z4z3j0E5mfRlPzxkySotzUHwRXjlpRh10vc6AI6WN62ehZj82VG7M20rqogJ0GLwar2Xa05a8Q==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lru.min@1.1.4: + resolution: {integrity: sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==} + engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lucia@3.2.2: resolution: {integrity: sha512-P1FlFBGCMPMXu+EGdVD9W4Mjm0DqsusmKgO7Xc33mI5X1bklmsQb0hfzPhXomQr9waWIBDsiOjvr1e6BTaUqpA==} deprecated: This package has been deprecated. Please see https://lucia-auth.com/lucia-v3/migrate. @@ -3383,6 +4127,10 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + mikro-orm@7.1.4: + resolution: {integrity: sha512-6e71y7tgSWNHMPCsTBxAaEaeBKly99fCYIOByrUVTAP/lm/N9iHYVEATeC7/oqVDYINDzdlJGKk9RI8objPl4A==} + engines: {node: '>= 22.17.0'} + mimic-fn@2.1.0: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} @@ -3469,10 +4217,20 @@ packages: resolution: {integrity: sha512-Qpu2ADfbKzyLdwC/5d4W7+5Yz7yBzCU05YWt5npWzACST37wJsB23wgOSo00qi043urkiRwXtEvJc9UnuLX/MQ==} engines: {node: '>= 8.0'} + mysql2@3.22.4: + resolution: {integrity: sha512-CtXYlmL7ZamiYKbmqkamQHWJROUHSfm+f3kByzGfknw7kW51mcB2ouMUqYq1XfYxbXmnWo6RhPydx6OCqdgcmQ==} + engines: {node: '>= 8.0'} + peerDependencies: + '@types/node': '>= 8' + named-placeholders@1.1.3: resolution: {integrity: sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==} engines: {node: '>=12.0.0'} + named-placeholders@1.1.6: + resolution: {integrity: sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==} + engines: {node: '>=8.0.0'} + nanoid@3.3.11: resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -3848,6 +4606,9 @@ packages: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + resolve@1.22.8: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} hasBin: true @@ -4003,6 +4764,13 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + source-map-support@0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + sparse-bitfield@3.0.3: resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} @@ -4019,6 +4787,10 @@ packages: sprintf-js@1.1.3: resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + sql-escaper@1.3.3: + resolution: {integrity: sha512-BsTCV265VpTp8tm1wyIm1xqQCS+Q9NHx2Sr+WcnUrgLrQ6yiDIvHYJV5gHxsj1lMBy2zm5twLaZao8Jd+S8JJw==} + engines: {bun: '>=1.0.0', deno: '>=2.0.0', node: '>=12.0.0'} + sql-formatter@15.4.9: resolution: {integrity: sha512-5vmt2HlCAVozxsBZuXWkAki/KGawaK+b5GG5x+BtXOFVpN/8cqppblFUxHl4jxdA0cvo14lABhM+KBnrUapOlw==} hasBin: true @@ -4191,6 +4963,11 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.23.0: + resolution: {integrity: sha512-eUdUIaCr963q2h5u3+QwvYp0+eqPvn+egeqZUm0hwERCqqx1E3kK5ehbGCvqSE5MQAULr67ww0cA3jKc3YkM1w==} + engines: {node: '>=18.0.0'} + hasBin: true + tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} @@ -5041,6 +5818,8 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 + '@drizzle-team/brocli@0.10.2': {} + '@emnapi/core@1.8.1': dependencies: '@emnapi/wasi-threads': 1.1.0 @@ -5073,84 +5852,316 @@ snapshots: tslib: 2.8.1 optional: true + '@esbuild-kit/core-utils@3.3.2': + dependencies: + esbuild: 0.18.20 + source-map-support: 0.5.21 + + '@esbuild-kit/esm-loader@2.6.5': + dependencies: + '@esbuild-kit/core-utils': 3.3.2 + get-tsconfig: 4.14.0 + + '@esbuild/aix-ppc64@0.25.12': + optional: true + '@esbuild/aix-ppc64@0.27.0': optional: true + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.18.20': + optional: true + + '@esbuild/android-arm64@0.25.12': + optional: true + '@esbuild/android-arm64@0.27.0': optional: true + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.18.20': + optional: true + + '@esbuild/android-arm@0.25.12': + optional: true + '@esbuild/android-arm@0.27.0': optional: true + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.18.20': + optional: true + + '@esbuild/android-x64@0.25.12': + optional: true + '@esbuild/android-x64@0.27.0': optional: true + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.18.20': + optional: true + + '@esbuild/darwin-arm64@0.25.12': + optional: true + '@esbuild/darwin-arm64@0.27.0': optional: true + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.18.20': + optional: true + + '@esbuild/darwin-x64@0.25.12': + optional: true + '@esbuild/darwin-x64@0.27.0': optional: true + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.18.20': + optional: true + + '@esbuild/freebsd-arm64@0.25.12': + optional: true + '@esbuild/freebsd-arm64@0.27.0': optional: true + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.18.20': + optional: true + + '@esbuild/freebsd-x64@0.25.12': + optional: true + '@esbuild/freebsd-x64@0.27.0': optional: true + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.18.20': + optional: true + + '@esbuild/linux-arm64@0.25.12': + optional: true + '@esbuild/linux-arm64@0.27.0': optional: true + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.18.20': + optional: true + + '@esbuild/linux-arm@0.25.12': + optional: true + '@esbuild/linux-arm@0.27.0': optional: true + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.18.20': + optional: true + + '@esbuild/linux-ia32@0.25.12': + optional: true + '@esbuild/linux-ia32@0.27.0': optional: true + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.18.20': + optional: true + + '@esbuild/linux-loong64@0.25.12': + optional: true + '@esbuild/linux-loong64@0.27.0': optional: true + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.18.20': + optional: true + + '@esbuild/linux-mips64el@0.25.12': + optional: true + '@esbuild/linux-mips64el@0.27.0': optional: true + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.18.20': + optional: true + + '@esbuild/linux-ppc64@0.25.12': + optional: true + '@esbuild/linux-ppc64@0.27.0': optional: true + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.18.20': + optional: true + + '@esbuild/linux-riscv64@0.25.12': + optional: true + '@esbuild/linux-riscv64@0.27.0': optional: true + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.18.20': + optional: true + + '@esbuild/linux-s390x@0.25.12': + optional: true + '@esbuild/linux-s390x@0.27.0': optional: true + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.18.20': + optional: true + + '@esbuild/linux-x64@0.25.12': + optional: true + '@esbuild/linux-x64@0.27.0': optional: true + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.25.12': + optional: true + '@esbuild/netbsd-arm64@0.27.0': optional: true + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.18.20': + optional: true + + '@esbuild/netbsd-x64@0.25.12': + optional: true + '@esbuild/netbsd-x64@0.27.0': optional: true + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.25.12': + optional: true + '@esbuild/openbsd-arm64@0.27.0': optional: true + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.18.20': + optional: true + + '@esbuild/openbsd-x64@0.25.12': + optional: true + '@esbuild/openbsd-x64@0.27.0': optional: true + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.25.12': + optional: true + '@esbuild/openharmony-arm64@0.27.0': optional: true + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.18.20': + optional: true + + '@esbuild/sunos-x64@0.25.12': + optional: true + '@esbuild/sunos-x64@0.27.0': optional: true + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.18.20': + optional: true + + '@esbuild/win32-arm64@0.25.12': + optional: true + '@esbuild/win32-arm64@0.27.0': optional: true + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.18.20': + optional: true + + '@esbuild/win32-ia32@0.25.12': + optional: true + '@esbuild/win32-ia32@0.27.0': optional: true + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.18.20': + optional: true + + '@esbuild/win32-x64@0.25.12': + optional: true + '@esbuild/win32-x64@0.27.0': optional: true + '@esbuild/win32-x64@0.28.1': + optional: true + '@fastify/ajv-compiler@4.0.5': dependencies: ajv: 8.18.0 @@ -5265,6 +6276,43 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 + '@mikro-orm/cli@7.1.4': + dependencies: + '@mikro-orm/core': 7.1.4 + mikro-orm: 7.1.4 + yargs: 17.7.2 + transitivePeerDependencies: + - dataloader + + '@mikro-orm/core@7.1.4': {} + + '@mikro-orm/migrations@7.1.4(@mikro-orm/core@7.1.4)': + dependencies: + '@mikro-orm/core': 7.1.4 + '@mikro-orm/sql': 7.1.4(@mikro-orm/core@7.1.4) + + '@mikro-orm/mysql@7.1.4(@mikro-orm/core@7.1.4)(@types/node@25.5.0)': + dependencies: + '@mikro-orm/core': 7.1.4 + '@mikro-orm/sql': 7.1.4(@mikro-orm/core@7.1.4) + kysely: 0.29.2 + mysql2: 3.22.4(@types/node@25.5.0) + sqlstring: 2.3.3 + transitivePeerDependencies: + - '@types/node' + + '@mikro-orm/sql@7.1.4(@mikro-orm/core@7.1.4)': + dependencies: + '@mikro-orm/core': 7.1.4 + kysely: 0.29.2 + + '@mikro-orm/sqlite@7.1.4(@mikro-orm/core@7.1.4)': + dependencies: + '@mikro-orm/core': 7.1.4 + '@mikro-orm/sql': 7.1.4(@mikro-orm/core@7.1.4) + better-sqlite3: 12.10.0 + kysely: 0.29.2 + '@mongodb-js/saslprep@1.3.1': dependencies: sparse-bitfield: 3.0.3 @@ -6039,6 +7087,10 @@ snapshots: '@types/async@3.2.24': {} + '@types/better-sqlite3@7.6.13': + dependencies: + '@types/node': 22.16.2 + '@types/body-parser@1.19.6': dependencies: '@types/connect': 3.4.38 @@ -6184,7 +7236,7 @@ snapshots: obug: 2.1.1 std-env: 4.0.0 tinyrainbow: 3.1.0 - vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) '@vitest/expect@4.1.5': dependencies: @@ -6195,21 +7247,21 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3))': + '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) - '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3))': + '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) '@vitest/pretty-format@4.1.5': dependencies: @@ -6238,7 +7290,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) '@vitest/utils@4.1.5': dependencies: @@ -6357,6 +7409,11 @@ snapshots: dependencies: is-windows: 1.0.2 + better-sqlite3@12.10.0: + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.3 + big-integer@1.6.51: {} big-integer@1.6.52: {} @@ -6365,6 +7422,10 @@ snapshots: binary-extensions@2.3.0: {} + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + bl@4.1.0: dependencies: buffer: 5.7.1 @@ -6403,6 +7464,8 @@ snapshots: buffer-equal-constant-time@1.0.1: {} + buffer-from@1.1.2: {} + buffer@5.7.1: dependencies: base64-js: 1.5.1 @@ -6585,6 +7648,22 @@ snapshots: dotenv@16.4.5: {} + drizzle-kit@0.31.10: + dependencies: + '@drizzle-team/brocli': 0.10.2 + '@esbuild-kit/esm-loader': 2.6.5 + esbuild: 0.25.12 + tsx: 4.23.0 + + drizzle-orm@0.45.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/pg@8.15.4)(better-sqlite3@12.10.0)(kysely@0.29.2)(mysql2@3.22.4(@types/node@25.5.0)): + optionalDependencies: + '@opentelemetry/api': 1.9.0 + '@types/better-sqlite3': 7.6.13 + '@types/pg': 8.15.4 + better-sqlite3: 12.10.0 + kysely: 0.29.2 + mysql2: 3.22.4(@types/node@25.5.0) + ebnf@1.9.1: {} ecdsa-sig-formatter@1.0.11: @@ -6606,6 +7685,60 @@ snapshots: es-module-lexer@2.0.0: {} + esbuild@0.18.20: + optionalDependencies: + '@esbuild/android-arm': 0.18.20 + '@esbuild/android-arm64': 0.18.20 + '@esbuild/android-x64': 0.18.20 + '@esbuild/darwin-arm64': 0.18.20 + '@esbuild/darwin-x64': 0.18.20 + '@esbuild/freebsd-arm64': 0.18.20 + '@esbuild/freebsd-x64': 0.18.20 + '@esbuild/linux-arm': 0.18.20 + '@esbuild/linux-arm64': 0.18.20 + '@esbuild/linux-ia32': 0.18.20 + '@esbuild/linux-loong64': 0.18.20 + '@esbuild/linux-mips64el': 0.18.20 + '@esbuild/linux-ppc64': 0.18.20 + '@esbuild/linux-riscv64': 0.18.20 + '@esbuild/linux-s390x': 0.18.20 + '@esbuild/linux-x64': 0.18.20 + '@esbuild/netbsd-x64': 0.18.20 + '@esbuild/openbsd-x64': 0.18.20 + '@esbuild/sunos-x64': 0.18.20 + '@esbuild/win32-arm64': 0.18.20 + '@esbuild/win32-ia32': 0.18.20 + '@esbuild/win32-x64': 0.18.20 + + esbuild@0.25.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.12 + '@esbuild/android-arm': 0.25.12 + '@esbuild/android-arm64': 0.25.12 + '@esbuild/android-x64': 0.25.12 + '@esbuild/darwin-arm64': 0.25.12 + '@esbuild/darwin-x64': 0.25.12 + '@esbuild/freebsd-arm64': 0.25.12 + '@esbuild/freebsd-x64': 0.25.12 + '@esbuild/linux-arm': 0.25.12 + '@esbuild/linux-arm64': 0.25.12 + '@esbuild/linux-ia32': 0.25.12 + '@esbuild/linux-loong64': 0.25.12 + '@esbuild/linux-mips64el': 0.25.12 + '@esbuild/linux-ppc64': 0.25.12 + '@esbuild/linux-riscv64': 0.25.12 + '@esbuild/linux-s390x': 0.25.12 + '@esbuild/linux-x64': 0.25.12 + '@esbuild/netbsd-arm64': 0.25.12 + '@esbuild/netbsd-x64': 0.25.12 + '@esbuild/openbsd-arm64': 0.25.12 + '@esbuild/openbsd-x64': 0.25.12 + '@esbuild/openharmony-arm64': 0.25.12 + '@esbuild/sunos-x64': 0.25.12 + '@esbuild/win32-arm64': 0.25.12 + '@esbuild/win32-ia32': 0.25.12 + '@esbuild/win32-x64': 0.25.12 + esbuild@0.27.0: optionalDependencies: '@esbuild/aix-ppc64': 0.27.0 @@ -6635,6 +7768,35 @@ snapshots: '@esbuild/win32-ia32': 0.27.0 '@esbuild/win32-x64': 0.27.0 + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + escalade@3.1.2: {} esprima@4.0.1: {} @@ -6758,6 +7920,8 @@ snapshots: dependencies: is-unicode-supported: 2.1.0 + file-uri-to-path@1.0.0: {} + fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -6817,6 +7981,10 @@ snapshots: '@sec-ant/readable-stream': 0.4.1 is-stream: 4.0.1 + get-tsconfig@4.14.0: + dependencies: + resolve-pkg-maps: 1.0.0 + github-from-package@0.0.0: {} glob-parent@5.1.2: @@ -7055,6 +8223,8 @@ snapshots: kuler@2.0.0: {} + kysely@0.29.2: {} + leven@3.1.0: {} light-my-request@6.6.0: @@ -7152,6 +8322,8 @@ snapshots: long@5.2.3: {} + long@5.3.2: {} + lossless-json@2.0.11: {} lru-cache@10.4.3: {} @@ -7160,6 +8332,8 @@ snapshots: lru.min@1.1.1: {} + lru.min@1.1.4: {} + lucia@3.2.2: dependencies: '@oslojs/crypto': 1.0.1 @@ -7196,6 +8370,8 @@ snapshots: braces: 3.0.3 picomatch: 2.3.2 + mikro-orm@7.1.4: {} + mimic-fn@2.1.0: {} mimic-function@5.0.1: {} @@ -7262,10 +8438,26 @@ snapshots: seq-queue: 0.0.5 sqlstring: 2.3.3 + mysql2@3.22.4(@types/node@25.5.0): + dependencies: + '@types/node': 25.5.0 + aws-ssl-profiles: 1.1.2 + denque: 2.1.0 + generate-function: 2.3.1 + iconv-lite: 0.7.2 + long: 5.3.2 + lru.min: 1.1.4 + named-placeholders: 1.1.6 + sql-escaper: 1.3.3 + named-placeholders@1.1.3: dependencies: lru-cache: 7.18.3 + named-placeholders@1.1.6: + dependencies: + lru.min: 1.1.1 + nanoid@3.3.11: {} napi-build-utils@2.0.0: {} @@ -7647,6 +8839,8 @@ snapshots: resolve-from@5.0.0: {} + resolve-pkg-maps@1.0.0: {} + resolve@1.22.8: dependencies: is-core-module: 2.14.0 @@ -7808,6 +9002,13 @@ snapshots: source-map-js@1.2.1: {} + source-map-support@0.5.21: + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + + source-map@0.6.1: {} + sparse-bitfield@3.0.3: dependencies: memory-pager: 1.5.0 @@ -7823,6 +9024,8 @@ snapshots: sprintf-js@1.1.3: {} + sql-escaper@1.3.3: {} + sql-formatter@15.4.9: dependencies: argparse: 2.0.1 @@ -7985,6 +9188,12 @@ snapshots: tslib@2.8.1: {} + tsx@4.23.0: + dependencies: + esbuild: 0.28.1 + optionalDependencies: + fsevents: 2.3.3 + tunnel-agent@0.6.0: dependencies: safe-buffer: 5.2.1 @@ -8027,7 +9236,7 @@ snapshots: vary@1.1.2: {} - vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3): + vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -8036,11 +9245,12 @@ snapshots: tinyglobby: 0.2.16 optionalDependencies: '@types/node': 22.16.2 - esbuild: 0.27.0 + esbuild: 0.28.1 fsevents: 2.3.3 + tsx: 4.23.0 yaml: 2.8.3 - vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3): + vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -8049,14 +9259,15 @@ snapshots: tinyglobby: 0.2.16 optionalDependencies: '@types/node': 25.5.0 - esbuild: 0.27.0 + esbuild: 0.28.1 fsevents: 2.3.3 + tsx: 4.23.0 yaml: 2.8.3 - vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)): + vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.5 - '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) + '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) '@vitest/pretty-format': 4.1.5 '@vitest/runner': 4.1.5 '@vitest/snapshot': 4.1.5 @@ -8073,7 +9284,7 @@ snapshots: tinyexec: 1.0.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@22.16.2)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 @@ -8083,10 +9294,10 @@ snapshots: transitivePeerDependencies: - msw - vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)): + vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.5 - '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) + '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3)) '@vitest/pretty-format': 4.1.5 '@vitest/runner': 4.1.5 '@vitest/snapshot': 4.1.5 @@ -8103,7 +9314,7 @@ snapshots: tinyexec: 1.0.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@25.5.0)(esbuild@0.28.1)(tsx@4.23.0)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7cba9ccfc..68489af56 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -19,9 +19,11 @@ allowBuilds: # These need install scripts to download/build native modules '@mongodb-js/zstd': true esbuild: false + sqlite3: true # We don't need to run these checks: https://github.com/protobufjs/protobuf.js/blob/master/scripts/postinstall.js protobufjs: false + better-sqlite3: true overrides: # Override to remove vitest production dependency diff --git a/scripts/compare-storage-benchmarks.mts b/scripts/compare-storage-benchmarks.mts new file mode 100644 index 000000000..e60c42205 --- /dev/null +++ b/scripts/compare-storage-benchmarks.mts @@ -0,0 +1,451 @@ +import inquirer from 'inquirer'; +import { spawn } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const OUTPUT_ENV = 'POWERSYNC_STORAGE_BENCHMARK_OUTPUT'; + +interface StorageBenchmarkResult { + storage: string; + version: number | null; + scenario: string; + source_rows: number; + buckets: number; + write_ms: number; + write_rows_per_second: number; + write_mebibytes_per_second: number; + sync_drain_ms: number; + ops: number; + read_mebibytes_per_second: number; +} + +interface StorageBenchmarkOutput { + schema_version: 1; + generated_at: string; + results: StorageBenchmarkResult[]; +} + +interface StorageRunner { + id: string; + label: string; + packageName: string; + testName?: string; +} + +const STORAGE_RUNNERS: StorageRunner[] = [ + { + id: 'mongodb', + label: 'MongoDB', + packageName: '@powersync/service-module-mongodb-storage' + }, + { + id: 'postgresql', + label: 'PostgreSQL', + packageName: '@powersync/service-module-postgres-storage' + }, + { + id: 'drizzle-sqlite', + label: 'Drizzle SQLite', + packageName: '@powersync/service-module-drizzle-storage' + }, + { + id: 'mikroorm-sqlite', + label: 'MikroORM SQLite', + packageName: '@powersync/service-module-mikroorm-storage', + testName: 'MikroORM SQLite' + }, + { + id: 'mikroorm-mysql', + label: 'MikroORM MySQL', + packageName: '@powersync/service-module-mikroorm-storage', + testName: 'MikroORM MySQL' + } +]; + +const STORAGE_ALIASES = new Map([ + ['mongo', 'mongodb'], + ['postgres', 'postgresql'], + ['drizzle:sqlite', 'drizzle-sqlite'], + ['mikroorm:sqlite', 'mikroorm-sqlite'], + ['mikroorm:mysql', 'mikroorm-mysql'] +]); + +interface CliOptions { + storageIds: string[]; + outputPath?: string; + chartPath: string; + help: boolean; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + printHelp(); + return; + } + + const selectedIds = options.storageIds.length > 0 ? options.storageIds : await promptForStorageIds(); + const runners = selectRunners(selectedIds); + const temporaryDirectory = await mkdtemp(join(tmpdir(), 'powersync-storage-benchmarks-')); + const results: StorageBenchmarkResult[] = []; + const failures: string[] = []; + + try { + for (const runner of runners) { + const resultPath = join(temporaryDirectory, `${runner.id}.json`); + console.log(`\nRunning ${runner.label} storage benchmarks...\n`); + + const exitCode = await runStorageBenchmark(runner, resultPath); + if (exitCode != 0) { + failures.push(`${runner.label} (exit code ${exitCode})`); + continue; + } + + try { + const output = parseBenchmarkOutput(await readFile(resultPath, 'utf8'), resultPath); + results.push(...output.results); + } catch (error) { + failures.push(`${runner.label} (${error instanceof Error ? error.message : String(error)})`); + } + } + + if (results.length > 0) { + console.log(`\nStorage benchmark comparison\n\n${formatComparison(results)}`); + await writeComparisonChart(options.chartPath, results); + console.log(`\nWrote benchmark comparison chart to ${options.chartPath}`); + } + + if (options.outputPath != null) { + const output: StorageBenchmarkOutput = { + schema_version: 1, + generated_at: new Date().toISOString(), + results + }; + await mkdir(dirname(options.outputPath), { recursive: true }); + await writeFile(options.outputPath, `${JSON.stringify(output, null, 2)}\n`, 'utf8'); + console.log(`\nWrote combined benchmark results to ${options.outputPath}`); + } + + if (failures.length > 0) { + console.error(`\nFailed benchmark runs:\n${failures.map((failure) => `- ${failure}`).join('\n')}`); + process.exitCode = 1; + } + } finally { + await rm(temporaryDirectory, { recursive: true, force: true }); + } +} + +function parseArgs(args: string[]): CliOptions { + const storageIds: string[] = []; + let outputPath: string | undefined; + let chartPath = 'storage-benchmark-comparison.svg'; + let help = false; + + for (let index = 0; index < args.length; index++) { + const argument = args[index]; + if (argument == '--help' || argument == '-h') { + help = true; + } else if (argument == '--storage' || argument == '-s') { + storageIds.push(...readOptionValue(args, ++index, argument).split(',')); + } else if (argument.startsWith('--storage=')) { + storageIds.push(...argument.slice('--storage='.length).split(',')); + } else if (argument == '--output' || argument == '-o') { + outputPath = readOptionValue(args, ++index, argument); + } else if (argument.startsWith('--output=')) { + outputPath = argument.slice('--output='.length); + } else if (argument == '--chart' || argument == '-c') { + chartPath = readOptionValue(args, ++index, argument); + } else if (argument.startsWith('--chart=')) { + chartPath = argument.slice('--chart='.length); + } else { + throw new Error(`Unknown option: ${argument}`); + } + } + + return { + storageIds: storageIds.map((id) => id.trim()).filter(Boolean), + outputPath, + chartPath, + help + }; +} + +function readOptionValue(args: string[], index: number, option: string) { + const value = args[index]; + if (value == null || value.startsWith('-')) { + throw new Error(`Missing value for ${option}`); + } + return value; +} + +async function promptForStorageIds() { + const answer = await inquirer.prompt<{ storageIds: string[] }>([ + { + type: 'checkbox', + name: 'storageIds', + message: 'Which bucket storage implementations should be compared?', + choices: STORAGE_RUNNERS.map((runner) => ({ + name: runner.label, + value: runner.id, + checked: runner.id == 'drizzle-sqlite' || runner.id == 'mikroorm-sqlite' + })), + validate: (selected: string[]) => selected.length > 0 || 'Select at least one storage implementation' + } + ]); + return answer.storageIds; +} + +function selectRunners(storageIds: string[]) { + const normalizedIds = storageIds.flatMap((id) => { + if (id == 'all') { + return STORAGE_RUNNERS.map((runner) => runner.id); + } + return [STORAGE_ALIASES.get(id) ?? id]; + }); + const unknownIds = normalizedIds.filter((id) => !STORAGE_RUNNERS.some((runner) => runner.id == id)); + if (unknownIds.length > 0) { + throw new Error(`Unknown storage implementation(s): ${[...new Set(unknownIds)].join(', ')}`); + } + + return [...new Set(normalizedIds)].map((id) => STORAGE_RUNNERS.find((runner) => runner.id == id)!); +} + +function runStorageBenchmark(runner: StorageRunner, resultPath: string) { + const command = process.platform == 'win32' ? 'corepack.cmd' : 'corepack'; + const args = ['pnpm', '--filter', runner.packageName, 'test', 'test/src/storage_bench.test.ts', '--run']; + if (runner.testName != null) { + args.push('-t', runner.testName); + } + + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: fileURLToPath(new URL('..', import.meta.url)), + env: { + ...process.env, + [OUTPUT_ENV]: resultPath + }, + stdio: 'inherit' + }); + child.once('error', reject); + child.once('close', (code, signal) => { + if (signal != null) { + console.error(`${runner.label} benchmark terminated by ${signal}`); + } + resolve(code ?? 1); + }); + }); +} + +function parseBenchmarkOutput(json: string, source: string): StorageBenchmarkOutput { + const parsed = JSON.parse(json) as Partial; + if (parsed.schema_version != 1 || !Array.isArray(parsed.results)) { + throw new Error(`Invalid benchmark output in ${source}`); + } + return parsed as StorageBenchmarkOutput; +} + +function formatComparison(results: StorageBenchmarkResult[]) { + const fastestWrites = maximumByScenario(results, (result) => result.write_mebibytes_per_second); + const fastestDrains = maximumByScenario(results, (result) => result.read_mebibytes_per_second); + const scenarioOrder = new Map(); + for (const result of results) { + if (!scenarioOrder.has(result.scenario)) { + scenarioOrder.set(result.scenario, scenarioOrder.size); + } + } + const rows = [ + '| Scenario | Storage | Version | Write rows/s | Write MiB/s | Write relative | Read MiB/s | Read relative | Write ms | Sync drain ms |', + '| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |' + ]; + + for (const result of [...results].sort((left, right) => compareResults(left, right, scenarioOrder))) { + rows.push( + `| ${result.scenario} | ${result.storage} | ${result.version ?? ''} | ${formatNumber( + result.write_rows_per_second + )} | ${formatNumber(result.write_mebibytes_per_second)} | ${formatPercent( + result.write_mebibytes_per_second, + fastestWrites.get(result.scenario)! + )} | ${formatNumber(result.read_mebibytes_per_second)} | ${formatPercent( + result.read_mebibytes_per_second, + fastestDrains.get(result.scenario)! + )} | ${formatNumber(result.write_ms)} | ${formatNumber(result.sync_drain_ms)} |` + ); + } + return rows.join('\n'); +} + +function maximumByScenario(results: StorageBenchmarkResult[], value: (result: StorageBenchmarkResult) => number) { + const maximums = new Map(); + for (const result of results) { + maximums.set(result.scenario, Math.max(maximums.get(result.scenario) ?? 0, value(result))); + } + return maximums; +} + +function compareResults( + left: StorageBenchmarkResult, + right: StorageBenchmarkResult, + scenarioOrder: Map +) { + return ( + scenarioOrder.get(left.scenario)! - scenarioOrder.get(right.scenario)! || left.storage.localeCompare(right.storage) + ); +} + +function formatPercent(value: number, maximum: number) { + return maximum == 0 ? 'n/a' : `${((value / maximum) * 100).toFixed(1)}%`; +} + +function formatNumber(value: number) { + if (Number.isInteger(value)) { + return String(value); + } + return value.toFixed(value >= 100 ? 0 : 2); +} + +export async function writeComparisonChart(outputPath: string, results: StorageBenchmarkResult[]) { + const width = Math.max(960, results.length * 110 + 180); + const height = 620; + const margin = { top: 90, right: 40, bottom: 190, left: 90 }; + const plotWidth = width - margin.left - margin.right; + const plotHeight = height - margin.top - margin.bottom; + const maximum = niceCeiling( + Math.max(...results.flatMap((result) => [result.write_mebibytes_per_second, result.read_mebibytes_per_second])) + ); + const scenarioOrder = new Map(); + for (const result of results) { + if (!scenarioOrder.has(result.scenario)) { + scenarioOrder.set(result.scenario, scenarioOrder.size); + } + } + const orderedResults = [...results].sort((left, right) => compareResults(left, right, scenarioOrder)); + const groupWidth = plotWidth / orderedResults.length; + const barWidth = Math.min(30, Math.max(8, groupWidth * 0.32)); + const barGap = Math.min(8, groupWidth * 0.08); + const baseline = margin.top + plotHeight; + const elements: string[] = []; + + for (let tick = 0; tick <= 5; tick++) { + const value = (maximum * tick) / 5; + const y = baseline - (plotHeight * tick) / 5; + elements.push( + ``, + `${escapeXml( + formatNumber(value) + )}` + ); + } + + orderedResults.forEach((result, index) => { + const center = margin.left + groupWidth * (index + 0.5); + const writeHeight = (result.write_mebibytes_per_second / maximum) * plotHeight; + const readHeight = (result.read_mebibytes_per_second / maximum) * plotHeight; + const writeX = center - barGap / 2 - barWidth; + const readX = center + barGap / 2; + const label = `${result.storage} v${result.version ?? 'default'} ยท ${result.scenario}`; + elements.push( + chartBar(writeX, baseline - writeHeight, barWidth, writeHeight, '#2563eb', 'Write', result, label), + chartBar(readX, baseline - readHeight, barWidth, readHeight, '#f97316', 'Read', result, label), + `${escapeXml(label)}` + ); + }); + + const svg = ` + + Storage benchmark write and read throughput + Grouped bar chart comparing logical source write throughput and sync stream read throughput in mebibytes per second. + + + Storage benchmark throughput + Logical source writes vs. sync stream reads (MiB/s) + + Write + + Read + MiB/s + ${elements.join('\n ')} + + +`; + + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, svg, 'utf8'); +} + +function chartBar( + x: number, + y: number, + width: number, + height: number, + color: string, + metric: string, + result: StorageBenchmarkResult, + label: string +) { + const value = metric == 'Write' ? result.write_mebibytes_per_second : result.read_mebibytes_per_second; + return `${escapeXml( + `${label}: ${metric} ${formatNumber(value)} MiB/s` + )}`; +} + +function niceCeiling(value: number) { + if (!Number.isFinite(value) || value <= 0) { + return 1; + } + const magnitude = 10 ** Math.floor(Math.log10(value)); + const normalized = value / magnitude; + const nice = normalized <= 1 ? 1 : normalized <= 2 ? 2 : normalized <= 5 ? 5 : 10; + return nice * magnitude; +} + +function escapeXml(value: string) { + return value.replace(/[&<>"']/g, (character) => { + return { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''' + }[character]!; + }); +} + +function printHelp() { + console.log(`Compare sync bucket storage benchmarks. + +Usage: + pnpm benchmark:storage [options] + +Options: + -s, --storage Comma-separated storage IDs; repeatable. Use "all" for every implementation. + -o, --output Write the combined JSON results to this path. + -c, --chart Write the SVG chart to this path (default: storage-benchmark-comparison.svg). + -h, --help Show this help. + +Storage IDs: +${STORAGE_RUNNERS.map((runner) => ` ${runner.id.padEnd(18)} ${runner.label}`).join('\n')} + +Database-backed runners use their module's normal test connection environment variables.`); +} + +if (process.argv[1] != null && import.meta.url == pathToFileURL(resolve(process.argv[1])).href) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/service/Dockerfile b/service/Dockerfile index fbfeb2f3f..e214b0304 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -21,6 +21,8 @@ COPY modules/module-postgres/package.json modules/module-postgres/tsconfig.json COPY modules/module-postgres-storage/package.json modules/module-postgres-storage/tsconfig.json modules/module-postgres-storage/ COPY modules/module-mongodb/package.json modules/module-mongodb/tsconfig.json modules/module-mongodb/ COPY modules/module-mongodb-storage/package.json modules/module-mongodb-storage/tsconfig.json modules/module-mongodb-storage/ +COPY modules/module-mikroorm-storage/package.json modules/module-mikroorm-storage/tsconfig.json modules/module-mikroorm-storage/ +COPY modules/module-drizzle-storage/package.json modules/module-drizzle-storage/tsconfig.json modules/module-drizzle-storage/ COPY modules/module-mysql/package.json modules/module-mysql/tsconfig.json modules/module-mysql/ COPY modules/module-mssql/package.json modules/module-mssql/tsconfig.json modules/module-mssql/ COPY modules/module-convex/package.json modules/module-convex/tsconfig.json modules/module-convex/ @@ -50,13 +52,19 @@ COPY modules/module-postgres/sql modules/module-postgres/sql/ COPY modules/module-postgres-storage/src modules/module-postgres-storage/src/ COPY modules/module-mongodb/src modules/module-mongodb/src/ COPY modules/module-mongodb-storage/src modules/module-mongodb-storage/src/ +COPY modules/module-mikroorm-storage/src modules/module-mikroorm-storage/src/ +COPY modules/module-drizzle-storage/src modules/module-drizzle-storage/src/ COPY modules/module-mysql/src modules/module-mysql/src/ COPY modules/module-mssql/src modules/module-mssql/src/ COPY modules/module-convex/src modules/module-convex/src/ RUN pnpm build:production && \ rm -rf node_modules **/node_modules && \ - pnpm install --frozen-lockfile --prod --ignore-scripts + pnpm install --frozen-lockfile --prod --ignore-scripts && \ + cd modules/module-mikroorm-storage && \ + pnpm rebuild better-sqlite3 && \ + cd ../module-drizzle-storage && \ + pnpm rebuild better-sqlite3 # === PROD === diff --git a/service/package.json b/service/package.json index 4c803cec9..80f5f87be 100644 --- a/service/package.json +++ b/service/package.json @@ -16,6 +16,8 @@ "@powersync/service-module-postgres-storage": "workspace:*", "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-mikroorm-storage": "workspace:*", + "@powersync/service-module-drizzle-storage": "workspace:*", "@powersync/service-module-mssql": "workspace:*", "@powersync/service-module-mysql": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", diff --git a/service/src/util/modules.ts b/service/src/util/modules.ts index 45631cd7e..f5d10b980 100644 --- a/service/src/util/modules.ts +++ b/service/src/util/modules.ts @@ -11,6 +11,12 @@ export const DYNAMIC_MODULES: core.ModuleLoaders = { storage: { mongodb: () => import('@powersync/service-module-mongodb-storage').then((module) => new module.MongoStorageModule()), + 'mikroorm:sqlite': () => + import('@powersync/service-module-mikroorm-storage').then((module) => new module.MikroOrmStorageModule()), + 'mikroorm:mysql': () => + import('@powersync/service-module-mikroorm-storage').then((module) => new module.MikroOrmStorageModule()), + 'drizzle:sqlite': () => + import('@powersync/service-module-drizzle-storage').then((module) => new module.DrizzleStorageModule()), postgresql: () => import('@powersync/service-module-postgres-storage').then((module) => new module.PostgresStorageModule()) } diff --git a/service/tsconfig.json b/service/tsconfig.json index 477b55a3e..f4a78412d 100644 --- a/service/tsconfig.json +++ b/service/tsconfig.json @@ -30,6 +30,9 @@ { "path": "../modules/module-postgres-storage" }, + { + "path": "../modules/module-drizzle-storage" + }, { "path": "../modules/module-mysql" }, diff --git a/tsconfig.json b/tsconfig.json index e01370b9f..bdd14875f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -34,6 +34,12 @@ { "path": "./modules/module-postgres-storage" }, + { + "path": "./modules/module-mikroorm-storage" + }, + { + "path": "./modules/module-drizzle-storage" + }, { "path": "./modules/module-mssql" }, @@ -97,6 +103,12 @@ { "path": "./modules/module-postgres-storage/test" }, + { + "path": "./modules/module-mikroorm-storage/test" + }, + { + "path": "./modules/module-drizzle-storage/test" + }, { "path": "./modules/module-mssql/test" },