diff --git a/dist/index.js b/dist/index.js index ca7c192b..83272b5f 100644 --- a/dist/index.js +++ b/dist/index.js @@ -2478,6 +2478,7 @@ const memoryLanceDBProPlugin = { mdMirror, workspaceBoundary: config.workspaceBoundary, selfImprovementMaxEntries: config.selfImprovement?.maxEntries, + manualStoreSupersede: config.manualStoreSupersede === true, // Mirrors the CLI context wiring below: keep in-process reflection caches // consistent after a live memory_forget delete too, not just CLI delete/delete-bulk. onMemoriesDeleted: ({ scopeFilter }) => invalidateReflectionCachesAfterDelete(scopeFilter), @@ -5236,6 +5237,7 @@ export function parsePluginConfig(value) { batchChunkSize: (() => { const raw = parsePositiveInt(cfg.batchChunkSize); return raw === undefined ? undefined : Math.min(50, raw); })(), scopes: typeof cfg.scopes === "object" && cfg.scopes !== null ? cfg.scopes : undefined, enableManagementTools: cfg.enableManagementTools === true, + manualStoreSupersede: cfg.manualStoreSupersede === true, sessionStrategy, selfImprovement: typeof cfg.selfImprovement === "object" && cfg.selfImprovement !== null ? { diff --git a/dist/src/store.js b/dist/src/store.js index 9ce524c9..dd928240 100644 --- a/dist/src/store.js +++ b/dist/src/store.js @@ -1856,6 +1856,58 @@ export class MemoryStore { .sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0)) .slice(offset, offset + limit); } + /** + * Bounded candidate scan for fact-key collision discovery. Unlike list(), + * which materializes and sorts the entire matching scope on every call, + * this pushes a hard row limit into the database query and never sorts. + * Deliberately NO content-based narrowing: an effective fact key can come + * from an explicit metadata field in any valid JSON layout (spaced colons, + * unicode escapes), from a stamped memory category in the same layouts, or + * be derived from the legacy storage category column plus row text when + * metadata is empty — so any serialization-layout pattern (LIKE on raw + * JSON) can exclude a valid candidate and silently break the collision + * set's completeness. Completeness therefore comes from the scope itself: + * every in-scope row is a candidate, the caller's bound keeps the scan + * finite, and an over-bound scope is an explicit rejection rather than a + * narrowed guess. Exact normalized-key comparison and active-row filtering + * stay with the caller. + * Returns at most bound + 1 rows so the caller can detect an over-bound + * candidate set without this method ever fetching an unbounded one. + */ + async listFactKeyCandidates(scopeFilter, bound) { + await this.ensureInitialized(); + if (isExplicitDenyAllScopeFilter(scopeFilter)) + return []; + const conditions = []; + if (scopeFilter.length > 0) { + const scopeConditions = scopeFilter + .map((scope) => `scope = '${escapeSqlLiteral(scope)}'`) + .join(" OR "); + conditions.push(`(${scopeConditions})`); + } + const applyConditions = (query) => conditions.length > 0 + ? query.where(conditions.join(" AND ")).limit(bound + 1) + : query.limit(bound + 1); + const results = await this.queryRowsWithProjectionFallback(applyConditions, [ + "id", + "text", + "category", + "scope", + "importance", + "timestamp", + "metadata", + ]); + return results.map((row) => ({ + id: row.id, + text: row.text, + vector: [], + category: row.category, + scope: row.scope ?? "global", + importance: clampImportance(Number(row.importance)), + timestamp: normalizeMemoryTimestamp(row.timestamp, 0), + metadata: row.metadata || "{}", + })); + } async queryRowsWithProjectionFallback(applyFilters, columns) { const projectedRows = await applyFilters(this.table.query()) .select(columns) @@ -2346,8 +2398,9 @@ export class MemoryStore { } /** * The locked body of update(). Callers must already hold the write lock - * and the serialized-update slot; transformMetadata composes it with a - * fresh read-decide step under the same lock. + * and the serialized-update slot: update() wraps it, transformMetadata + * composes it with a fresh read-decide step under the same lock, and the + * supersede commit path calls it from inside its own atomic section. */ async performUpdateLocked(id, updates, scopeFilter) { // Support full UUID, short hex prefixes, and constrained exact legacy IDs imported @@ -2664,6 +2717,86 @@ export class MemoryStore { this.noteDataModification(); return { repaired, failed, skipped, unrecovered }; } + /** + * Force the open table handle onto the latest committed version. A nonzero + * readConsistencyInterval lets reads serve a snapshot up to that many + * seconds stale; a locked read-modify-write section must observe every + * commit that preceded its lock acquisition, so it re-syncs first. A sync + * failure propagates: proceeding on a possibly-stale snapshot would + * silently reintroduce the staleness this guard exists to close. + */ + async syncTableToLatest() { + const table = this.table; + if (table && typeof table.checkoutLatest === "function") { + await table.checkoutLatest(); + } + } + /** + * Atomic supersede-and-store: re-discovers the target rows, inserts the new + * row, and invalidates every confirmed target inside ONE write-lock + + * serialized-update section. The caller's advisory discovery only decides + * whether to enter this path; the target set that actually commits is the + * one discovered here, so two concurrent same-key writers converge on a + * single active row (the second writer's recheck sees the first writer's + * replacement and supersedes it) instead of leaving both replacements + * standing. + * + * Only CONFIRMED invalidations are reported in supersededIds; a null or + * throwing patch lands in invalidationFailures instead of being silently + * counted as success. + */ + async storeSuperseding(options) { + await this.ensureInitialized(); + const result = await this.runWithWriteLock(() => this.runSerializedUpdate(async () => { + // The cross-process lock serializes writers but does not refresh this + // handle's read snapshot: with a second store instance and a nonzero + // readConsistencyInterval, the locked re-discovery could miss the + // preceding writer's commit and leave both replacements active. + await this.syncTableToLatest(); + const targets = await options.discoverTargets(); + const fullEntry = { + ...options.entry, + id: randomUUID(), + timestamp: Date.now(), + metadata: options.finalizeEntryMetadata + ? options.finalizeEntryMetadata(targets) + : options.entry.metadata || "{}", + importance: clampImportance(Number(options.entry.importance)), + }; + await this.table.add([fullEntry]); + const supersededIds = []; + const invalidationFailures = []; + for (const target of targets) { + try { + const existing = await this.getById(target.id, options.scopeFilter); + if (!existing) { + invalidationFailures.push({ + id: target.id, + reason: "row not found or outside accessible scopes at commit time", + }); + continue; + } + const metadata = buildSmartMetadata(existing, options.buildTargetPatch(existing, fullEntry.id)); + const updated = await this.performUpdateLocked(target.id, { metadata: stringifySmartMetadata(metadata) }, options.scopeFilter); + if (updated == null) { + invalidationFailures.push({ id: target.id, reason: "update persisted no row" }); + } + else { + supersededIds.push(target.id); + } + } + catch (err) { + invalidationFailures.push({ + id: target.id, + reason: err instanceof Error ? err.message : String(err), + }); + } + } + return { entry: fullEntry, supersededIds, invalidationFailures }; + })); + this.noteDataModification(); + return result; + } async bulkDelete(scopeFilter, beforeTimestamp) { await this.ensureInitialized(); const conditions = []; diff --git a/dist/src/tools.js b/dist/src/tools.js index c5d9572f..7d0b11c2 100644 --- a/dist/src/tools.js +++ b/dist/src/tools.js @@ -167,10 +167,21 @@ function parseMetadataObject(rawMetadata) { function hasExplicitMetadataField(entry, field) { return Object.prototype.hasOwnProperty.call(parseMetadataObject(entry.metadata), field); } +/** + * The one normalization rule for comparing fact keys: explicit keys are stored + * trimmed but case-preserved while derived keys are lowercased, so every + * comparison site must apply the same trim+lowercase rule or a mixed-case + * explicit key ("Preferences:Theme") diverges from its query-equivalent + * derived form ("preferences:theme"). + */ +function normalizeFactKeyForComparison(key) { + const normalized = key?.trim().toLowerCase(); + return normalized ? normalized : undefined; +} function factQueryMatches(entry, query, factKey) { const meta = parseSmartMetadata(entry.metadata, entry); - const normalizedFactKey = factKey?.trim().toLowerCase(); - if (normalizedFactKey && meta.fact_key?.toLowerCase() !== normalizedFactKey) + const normalizedFactKey = normalizeFactKeyForComparison(factKey); + if (normalizedFactKey && normalizeFactKeyForComparison(meta.fact_key) !== normalizedFactKey) return false; const normalizedQuery = query?.trim().toLowerCase(); if (!normalizedQuery) @@ -212,6 +223,66 @@ function serializeFactEntry(entry, atMs) { }; } const FACT_QUERY_PAGE_SIZE = 500; +/** + * Hard ceiling on candidate rows a fact-key collision scan will accept from + * the store's bounded candidate query (see + * MemoryStore.listFactKeyCandidates). The scan runs inside + * storeSuperseding()'s cross-process write lock, so its work must stay + * bounded; a candidate set past this ceiling cannot guarantee a complete + * collision set, and an opted-in supersede write is REJECTED explicitly + * (FactKeyScanOverBoundError) rather than silently leaving the old same-key + * value active alongside the new one. + */ +const FACT_KEY_SCAN_MAX_ROWS = 20_000; +/** + * An opted-in manual supersede write was rejected because the fact-key + * collision scan could not be completed within its bound. Callers surface + * this to the agent explicitly; `force: true` stores without superseding. + */ +export class FactKeyScanOverBoundError extends Error { + constructor(bound) { + super(`manual supersede rejected: the scope holds more than ${bound} fact-key candidate rows, so a complete collision scan cannot be guaranteed. Retry with force: true to store without superseding, or reduce the scope's row count.`); + this.name = "FactKeyScanOverBoundError"; + } +} +/** + * Complete, scope-aware lookup of ACTIVE rows holding a fact key. The vector + * top-K is the wrong tool for key collisions: a same-key row ranked behind + * closer unrelated neighbors, or below the similarity floor, is exactly the + * stale value a manual update must supersede. The store performs a bounded, + * scope-only candidate query (hard limit push-down, no sort, no content + * narrowing: effective keys hide in any valid JSON layout or derive from the + * storage category column, so a serialization-layout pattern can exclude a + * valid candidate); exact normalized-key and active-row filtering happen + * here. An over-bound candidate set throws FactKeyScanOverBoundError — + * never a silently incomplete collision set. + */ +async function findActiveFactKeyEntries(store, scopeFilter, factKey) { + const matches = []; + const targetKey = normalizeFactKeyForComparison(factKey); + if (!targetKey) + return matches; + const now = Date.now(); + const rows = await store.listFactKeyCandidates(scopeFilter, FACT_KEY_SCAN_MAX_ROWS); + if (rows.length > FACT_KEY_SCAN_MAX_ROWS) { + throw new FactKeyScanOverBoundError(FACT_KEY_SCAN_MAX_ROWS); + } + for (const entry of rows) { + // Exact scope match: store reads mask legacy NULL scopes as "global" and + // store-level filters have passed legacy rows through before; a + // supersede scan must never treat a row outside the requested scopes + // as a collision target. + if (!scopeFilter.includes(entry.scope)) + continue; + const meta = parseSmartMetadata(entry.metadata, entry); + if (!isMemoryActiveAt(meta, now) || isMemoryExpired(meta, now)) + continue; + const entryKey = normalizeFactKeyForComparison(meta.fact_key ?? deriveFactKey(meta.memory_category, entry.text)); + if (entryKey === targetKey) + matches.push(entry); + } + return matches; +} function compareFactQueryCandidates(a, b) { if (a.fact.activeAt !== b.fact.activeAt) return a.fact.activeAt ? -1 : 1; @@ -1010,17 +1081,106 @@ export function registerMemoryStoreTool(api, context) { // Align with TEMPORAL_VERSIONED_CATEGORIES at the smart-category // layer so legacy storage categories like "fact" don't cross-match // unrelated profile/case memories. - let existing = []; + const manualSupersede = runtimeContext.manualStoreSupersede === true; + const newFactKey = deriveFactKey(memoryCategory, stripped); + // One discovery routine serves both passes: the ADVISORY pass below + // decides whether to enter the supersede path at all, and the store's + // atomic commit re-runs it under the write lock so the target set + // that actually commits reflects concurrent writers' work. + const runSupersedeDiscovery = async () => { + // Check for duplicates / supersede candidates using raw vector + // similarity (bypasses importance/recency weighting). + // Fail-open by design: dedup must never block a legitimate write. + // excludeInactive: superseded historical records must not block + // new writes. + let neighbors = []; + try { + neighbors = await runtimeContext.store.vectorSearch(vector, 3, 0.1, [ + targetScope, + ], { excludeInactive: true }); + } + catch (err) { + console.warn(`memory-lancedb-pro: duplicate pre-check failed, continue store: ${String(err)}`); + } + const duplicateCandidate = neighbors[0]?.score > 0.98 ? neighbors[0] : undefined; + // Key collisions resolve through a COMPLETE scope lookup, never the + // vector top-K: a same-key row ranked behind closer unrelated + // neighbors, or below the similarity floor, is exactly the stale + // value this path exists to supersede. Fail-open like the duplicate + // pre-check: a lookup error stores alongside instead of blocking. + let activeFactKeyEntries = []; + if (manualSupersede && !force && newFactKey) { + try { + activeFactKeyEntries = await findActiveFactKeyEntries(runtimeContext.store, [targetScope], newFactKey); + } + catch (err) { + // An over-bound candidate set is NOT the fail-open class: a + // store-alongside here silently abandons the supersession + // invariant, so the write is rejected explicitly instead. + if (err instanceof FactKeyScanOverBoundError) { + throw err; + } + console.warn(`memory-lancedb-pro: fact-key lookup failed, continue store: ${String(err)}`); + } + } + const manualPriorityTargets = []; + if (manualSupersede && !force) { + if (duplicateCandidate) { + manualPriorityTargets.push(duplicateCandidate); + } + for (const entry of activeFactKeyEntries) { + if (!manualPriorityTargets.some((target) => target.entry.id === entry.id)) { + manualPriorityTargets.push({ entry }); + } + } + } + // Auto-supersede band: similar memory (0.95-0.98), same + // storage-layer category, eligible category. + const bandCandidate = neighbors.find((r) => r.score > 0.95 && + r.score <= 0.98 && + TEMPORAL_VERSIONED_CATEGORIES.has(memoryCategory) && + matchesMemoryCategoryFilter(r.entry.category, memoryCategory, r.entry.metadata)); + const targets = manualPriorityTargets.length > 0 + ? manualPriorityTargets + : bandCandidate + ? [bandCandidate] + : []; + return { + neighbors, + duplicateCandidate, + manual: manualPriorityTargets.length > 0, + targets, + }; + }; + // An over-bound fact-key scan rejects the opted-in write explicitly: + // storing alongside would silently leave the old same-key value + // active, and pretending "created" hides exactly the failure the + // supersede contract exists to prevent. force: true bypasses the + // scan entirely and stores without superseding. + const buildOverBoundRejection = (err) => ({ + content: [ + { + type: "text", + text: err.message, + }, + ], + details: { + action: "rejected", + reason: "fact-key-scan-over-bound", + }, + }); + let discovery; try { - existing = await runtimeContext.store.vectorSearch(vector, 3, 0.1, [ - targetScope, - ], { excludeInactive: true }); + discovery = await runSupersedeDiscovery(); } catch (err) { - console.warn(`memory-lancedb-pro: duplicate pre-check failed, continue store: ${String(err)}`); + if (err instanceof FactKeyScanOverBoundError) { + return buildOverBoundRejection(err); + } + throw err; } - const duplicateCandidate = existing[0]?.score > 0.98 ? existing[0] : undefined; - if (duplicateCandidate && !force) { + const duplicateCandidate = discovery.duplicateCandidate; + if (duplicateCandidate && !force && !manualSupersede) { return { content: [ { @@ -1037,84 +1197,175 @@ export function registerMemoryStoreTool(api, context) { }, }; } - // Auto-supersede: if a similar memory exists (0.95-0.98 similarity), - // same storage-layer category, and category is eligible, mark the old - // one as superseded and store the new one with a supersedes link. - const supersedeCandidate = existing.find((r) => r.score > 0.95 && - r.score <= 0.98 && - TEMPORAL_VERSIONED_CATEGORIES.has(memoryCategory) && - matchesMemoryCategoryFilter(r.entry.category, memoryCategory, r.entry.metadata)); - if (supersedeCandidate) { - const oldEntry = supersedeCandidate.entry; - const oldMeta = parseSmartMetadata(oldEntry.metadata, oldEntry); - const now = Date.now(); - const factKey = oldMeta.fact_key ?? deriveFactKey(oldMeta.memory_category, text); - // Store new memory with supersedes link, preserving canonical fields - // from the old entry (aligns with memory_update supersede path). - const newMeta = buildSmartMetadata({ text, category: storageCategory, importance: safeImportance }, { - l0_abstract: text, - l1_overview: oldMeta.l1_overview || `- ${text}`, - l2_content: text, - memory_category: oldMeta.memory_category, - tier: oldMeta.tier, - source: "manual", - state: "confirmed", - memory_layer: deriveManualMemoryLayer(oldMeta.memory_category), - last_confirmed_use_at: now, - bad_recall_count: 0, - suppressed_until_turn: 0, - valid_from: now, - fact_key: factKey, - supersedes: oldEntry.id, - relations: appendRelation([], { - type: "supersedes", - targetId: oldEntry.id, - }), - }); - const newEntry = await runtimeContext.store.store({ - text, - vector, - importance: safeImportance, - category: storageCategory, - scope: targetScope, - metadata: stringifySmartMetadata(newMeta), - }); - // Invalidate old record - try { - await runtimeContext.store.patchMetadata(oldEntry.id, { - fact_key: factKey, - invalidated_at: now, - superseded_by: newEntry.id, - relations: appendRelation(oldMeta.relations, { + // Manual-priority supersede (manualStoreSupersede): a manual store + // always takes priority — its text lands verbatim, and a similar + // existing row yields to it. Targets, in order: the near-identical + // neighbor the duplicate check used to reject, then an active + // neighbor holding the same fact key at any similarity (the + // update/contradiction shape, e.g. a new value for a versioned + // fact). Anything else falls through to the versioned-band check, + // and past that stores alongside: a wrong supersede destroys a real + // fact, while a duplicate is fixable noise. + // A manual-priority write with an EMPTY advisory still enters the + // locked path: two first-time same-key writers otherwise both see + // nothing and both plain-store, leaving two active rows for one + // fact key. The locked rediscovery is authoritative; when it also + // finds nothing, the commit is a plain create. + const manualPriorityWrite = manualSupersede && !force; + if (discovery.targets.length > 0 || manualPriorityWrite) { + // Canonical identity comes from the REQUESTED store, never from a + // near-duplicate donor: the new row's category and fact key are the + // requested ones, and overview/tier inherit only from a + // category-verified target (the band shape). Temporal expiry is + // preserved exactly like the plain-store path. + const buildSupersedeMetadata = (targets, manualPriority) => { + const now = Date.now(); + const verified = targets + .map((target) => ({ target, meta: parseSmartMetadata(target.metadata, target) })) + .find(({ target }) => matchesMemoryCategoryFilter(target.category, memoryCategory, target.metadata)); + // The band keeps the verified target's ESTABLISHED key: deriving + // one from the replacement's wording would split the fact's + // history across two canonical identities. Requested-key + // precedence is the opt-in manual-priority contract only. + const factKey = manualPriority + ? newFactKey ?? verified?.meta.fact_key ?? undefined + : verified?.meta.fact_key ?? newFactKey ?? undefined; + const primary = targets[0]; + return stringifySmartMetadata(buildSmartMetadata({ text, category: storageCategory, importance: safeImportance }, { + l0_abstract: text, + // A manual-priority supersede replaces the fact's VALUE, so + // the old row's overview is stale by definition; the band + // case keeps the richer overview of its category-verified + // target as before. + l1_overview: manualPriority ? `- ${text}` : verified?.meta.l1_overview || `- ${text}`, + l2_content: text, + memory_category: memoryCategory, + tier: verified?.meta.tier, + source: "manual", + state: "confirmed", + memory_layer: deriveManualMemoryLayer(memoryCategory), + last_confirmed_use_at: now, + bad_recall_count: 0, + suppressed_until_turn: 0, + valid_from: now, + memory_temporal_type: temporalType, + valid_until: validUntil, + ...(factKey ? { fact_key: factKey } : {}), + ...(primary ? { supersedes: primary.id } : {}), + relations: targets.reduce((relations, target) => appendRelation(relations, { + type: "supersedes", + targetId: target.id, + }), []), + })); + }; + const buildInvalidationPatch = (target, newEntryId) => { + const targetMeta = parseSmartMetadata(target.metadata, target); + const sameCategory = matchesMemoryCategoryFilter(target.category, memoryCategory, target.metadata); + return { + // Backfill a missing fact key only on a category-verified + // target: stamping the new key onto a foreign-category + // near-duplicate would misfile it. + ...(!targetMeta.fact_key && sameCategory && newFactKey ? { fact_key: newFactKey } : {}), + invalidated_at: Date.now(), + superseded_by: newEntryId, + relations: appendRelation(targetMeta.relations, { type: "superseded_by", - targetId: newEntry.id, + targetId: newEntryId, }), - }, [targetScope]); + }; + }; + // Commit atomically at the store layer: recheck, insert, and + // invalidate run under one write lock, so concurrent same-key + // writers converge on a single active row instead of leaving two + // replacements standing. + let lastDiscovery = discovery; + let committed; + try { + committed = await runtimeContext.store.storeSuperseding({ + entry: { + text, + vector, + importance: safeImportance, + category: storageCategory, + scope: targetScope, + metadata: "{}", + }, + scopeFilter: [targetScope], + discoverTargets: async () => { + lastDiscovery = await runSupersedeDiscovery(); + return lastDiscovery.targets.map((target) => target.entry); + }, + finalizeEntryMetadata: (targets) => buildSupersedeMetadata(targets, lastDiscovery.manual), + buildTargetPatch: buildInvalidationPatch, + }); + } + catch (err) { + // The locked recheck hit the scan bound (the scope crossed it + // after the advisory pass): the write aborts before the insert, + // so rejecting here leaves no partial state behind. + if (err instanceof FactKeyScanOverBoundError) { + return buildOverBoundRejection(err); + } + throw err; } - catch (patchErr) { - // New record is already the source of truth; log but don't fail - console.warn(`memory-pro: failed to patch superseded record ${oldEntry.id.slice(0, 8)}: ${patchErr}`); + const newEntry = committed.entry; + const { supersededIds, invalidationFailures } = committed; + for (const failure of invalidationFailures) { + // The new record is already stored; surface the unconfirmed + // invalidation instead of silently reporting it as superseded. + console.warn(`memory-pro: failed to invalidate superseded record ${failure.id.slice(0, 8)}: ${failure.reason}`); } // Dual-write to Markdown mirror if enabled if (context.mdMirror) { await context.mdMirror({ text, category: storageCategory, scope: targetScope, timestamp: newEntry.timestamp }, { source: "memory_store", agentId }); } + if (lastDiscovery.targets.length === 0) { + // First writer of this fact: the authoritative locked discovery + // found nothing to supersede, so this commit was a plain create + // and must report as one, not as "superseded no memories". + return { + content: [ + { + type: "text", + text: `Stored: "${text.slice(0, 100)}${text.length > 100 ? "..." : ""}" in scope '${targetScope}'`, + }, + ], + details: { + action: "created", + id: newEntry.id, + scope: newEntry.scope, + category: memoryCategory, + rawCategory: newEntry.category, + importance: newEntry.importance, + }, + }; + } + const supersededLabel = supersededIds.length > 1 + ? `${supersededIds.length} memories (${supersededIds.map((id) => id.slice(0, 8)).join(", ")})` + : supersededIds.length === 1 + ? `memory ${supersededIds[0].slice(0, 8)}...` + : "no memories"; + const failureSuffix = invalidationFailures.length > 0 + ? ` (${invalidationFailures.length} invalidation(s) failed; those rows may still be active)` + : ""; return { content: [ { type: "text", - text: `Superseded memory ${oldEntry.id.slice(0, 8)}... → new version ${newEntry.id.slice(0, 8)}...: "${text.slice(0, 80)}${text.length > 80 ? "..." : ""}"`, + text: `Superseded ${supersededLabel} → new version ${newEntry.id.slice(0, 8)}...: "${text.slice(0, 80)}${text.length > 80 ? "..." : ""}"${failureSuffix}`, }, ], details: { action: "superseded", id: newEntry.id, - supersededId: oldEntry.id, + supersededId: supersededIds[0] ?? null, + supersededIds, + ...(invalidationFailures.length > 0 ? { invalidationFailures } : {}), scope: newEntry.scope, category: memoryCategory, rawCategory: newEntry.category, importance: newEntry.importance, - similarity: supersedeCandidate.score, + similarity: lastDiscovery.targets[0]?.score, }, }; } diff --git a/index.ts b/index.ts index 2938baad..67d8e365 100644 --- a/index.ts +++ b/index.ts @@ -277,6 +277,7 @@ interface PluginConfig { agentAccess?: Record; }; enableManagementTools?: boolean; + manualStoreSupersede?: boolean; sessionStrategy?: SessionStrategy; sessionMemory?: { enabled?: boolean; messageCount?: number }; selfImprovement?: { @@ -3326,6 +3327,7 @@ const memoryLanceDBProPlugin = { mdMirror, workspaceBoundary: config.workspaceBoundary, selfImprovementMaxEntries: config.selfImprovement?.maxEntries, + manualStoreSupersede: config.manualStoreSupersede === true, // Mirrors the CLI context wiring below: keep in-process reflection caches // consistent after a live memory_forget delete too, not just CLI delete/delete-bulk. onMemoriesDeleted: ({ scopeFilter }) => invalidateReflectionCachesAfterDelete(scopeFilter), @@ -6571,6 +6573,7 @@ export function parsePluginConfig(value: unknown): PluginConfig { batchChunkSize: (() => { const raw = parsePositiveInt(cfg.batchChunkSize); return raw === undefined ? undefined : Math.min(50, raw); })(), scopes: typeof cfg.scopes === "object" && cfg.scopes !== null ? cfg.scopes as any : undefined, enableManagementTools: cfg.enableManagementTools === true, + manualStoreSupersede: cfg.manualStoreSupersede === true, sessionStrategy, selfImprovement: typeof cfg.selfImprovement === "object" && cfg.selfImprovement !== null ? { diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 8376a5c2..f345c800 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -297,6 +297,11 @@ "default": false, "description": "Enable management/debug tools such as memory_list, memory_stats, and governance-oriented self-improvement review/extract actions" }, + "manualStoreSupersede": { + "type": "boolean", + "default": false, + "description": "Manual-priority store lane: a memory_store that finds a similar existing memory supersedes it instead of being rejected as a duplicate; the manual text always lands verbatim. Off by default to preserve the classic duplicate check." + }, "sessionStrategy": { "type": "string", "enum": [ @@ -2307,6 +2312,9 @@ "help": "Enable management/debug tools such as memory_list, memory_stats, and governance-oriented self-improvement review/extract actions.", "advanced": true }, + "manualStoreSupersede": { + "help": "Manual memory_store always lands; a similar existing memory is superseded by it instead of blocking the write (fact-key aware). Default off." + }, "mdMirror.enabled": { "label": "Markdown Mirror", "help": "Write a human-readable Markdown copy alongside LanceDB storage (dual-write mode)" diff --git a/package.json b/package.json index aa70fce0..bfe5fada 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/reflection-unattributed-session-read.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs && node --test test/smart-metadata-source-classification.test.mjs && node --test test/reflection-embed-transient-retry.test.mjs && node --test test/scope-owner-leak-hardening.test.mjs && node --test test/isOwnedByAgent.test.mjs && node --test test/typed-array-vector-fetch.test.mjs && node --test test/extraction-grounding-register.test.mjs && node test/grounding-rejudge.test.mjs && node --test test/reverse-map-legacy-category.test.mjs && node --test test/reflection-mapped-category-stamping.test.mjs && node --test test/memory-upgrader-category-normalization.test.mjs && node --test test/autocapture-fallback-gating.test.mjs && node --test test/prompt-architecture.test.mjs && node test/extraction-category-rubric.test.mjs && node --test test/admission-control-batch-utility.test.mjs && node --test test/smart-extractor-batch-admission.test.mjs && node --test test/admission-control-prompt-shape.test.mjs && node --test test/smart-extractor-merge-accounting.test.mjs && node --test test/admission-utility-veto.test.mjs && node --test test/cli-subcommand-attachment.test.mjs && node --test test/admission-lane-model-affinity.test.mjs && node --test test/admission-model-resolution.test.mjs && node --test test/admission-controller-standalone.test.mjs && node --test test/smart-extractor-admission-controller-injection.test.mjs && node --test test/admission-without-smart-extraction.test.mjs && node --test test/llm-thinklevel.test.mjs && node --test test/memory-id-prefix-resolution.test.mjs", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/reflection-unattributed-session-read.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs && node --test test/smart-metadata-source-classification.test.mjs && node --test test/reflection-embed-transient-retry.test.mjs && node --test test/scope-owner-leak-hardening.test.mjs && node --test test/isOwnedByAgent.test.mjs && node --test test/typed-array-vector-fetch.test.mjs && node --test test/extraction-grounding-register.test.mjs && node test/grounding-rejudge.test.mjs && node --test test/reverse-map-legacy-category.test.mjs && node --test test/reflection-mapped-category-stamping.test.mjs && node --test test/memory-upgrader-category-normalization.test.mjs && node --test test/autocapture-fallback-gating.test.mjs && node --test test/prompt-architecture.test.mjs && node test/extraction-category-rubric.test.mjs && node --test test/admission-control-batch-utility.test.mjs && node --test test/smart-extractor-batch-admission.test.mjs && node --test test/admission-control-prompt-shape.test.mjs && node --test test/smart-extractor-merge-accounting.test.mjs && node --test test/admission-utility-veto.test.mjs && node --test test/cli-subcommand-attachment.test.mjs && node --test test/admission-lane-model-affinity.test.mjs && node --test test/admission-model-resolution.test.mjs && node --test test/admission-controller-standalone.test.mjs && node --test test/smart-extractor-admission-controller-injection.test.mjs && node --test test/admission-without-smart-extraction.test.mjs && node --test test/llm-thinklevel.test.mjs && node --test test/memory-id-prefix-resolution.test.mjs && node --test test/manual-store-supersede.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index 199940b5..eb752982 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -139,6 +139,7 @@ export const CI_TEST_MANIFEST = [ { group: "core-regression", runner: "node", file: "test/admission-without-smart-extraction.test.mjs", args: ["--test"] }, { group: "cli-smoke", runner: "node", file: "test/cli-subcommand-attachment.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/memory-id-prefix-resolution.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/manual-store-supersede.test.mjs", args: ["--test"] }, ]; export function getEntriesForGroup(group) { diff --git a/src/store.ts b/src/store.ts index 88fbd549..826389f0 100644 --- a/src/store.ts +++ b/src/store.ts @@ -2250,6 +2250,71 @@ export class MemoryStore { .slice(offset, offset + limit); } + /** + * Bounded candidate scan for fact-key collision discovery. Unlike list(), + * which materializes and sorts the entire matching scope on every call, + * this pushes a hard row limit into the database query and never sorts. + * Deliberately NO content-based narrowing: an effective fact key can come + * from an explicit metadata field in any valid JSON layout (spaced colons, + * unicode escapes), from a stamped memory category in the same layouts, or + * be derived from the legacy storage category column plus row text when + * metadata is empty — so any serialization-layout pattern (LIKE on raw + * JSON) can exclude a valid candidate and silently break the collision + * set's completeness. Completeness therefore comes from the scope itself: + * every in-scope row is a candidate, the caller's bound keeps the scan + * finite, and an over-bound scope is an explicit rejection rather than a + * narrowed guess. Exact normalized-key comparison and active-row filtering + * stay with the caller. + * Returns at most bound + 1 rows so the caller can detect an over-bound + * candidate set without this method ever fetching an unbounded one. + */ + async listFactKeyCandidates( + scopeFilter: string[], + bound: number, + ): Promise { + await this.ensureInitialized(); + if (isExplicitDenyAllScopeFilter(scopeFilter)) return []; + + const conditions: string[] = []; + if (scopeFilter.length > 0) { + const scopeConditions = scopeFilter + .map((scope) => `scope = '${escapeSqlLiteral(scope)}'`) + .join(" OR "); + conditions.push(`(${scopeConditions})`); + } + + const applyConditions = (query: any) => + conditions.length > 0 + ? query.where(conditions.join(" AND ")).limit(bound + 1) + : query.limit(bound + 1); + + const results = await this.queryRowsWithProjectionFallback( + applyConditions, + [ + "id", + "text", + "category", + "scope", + "importance", + "timestamp", + "metadata", + ], + ); + + return results.map( + (row): MemoryEntry => ({ + id: row.id as string, + text: row.text as string, + vector: [], + category: row.category as MemoryEntry["category"], + scope: (row.scope as string | undefined) ?? "global", + importance: clampImportance(Number(row.importance)), + timestamp: normalizeMemoryTimestamp(row.timestamp, 0), + metadata: (row.metadata as string) || "{}", + }), + ); + } + private async queryRowsWithProjectionFallback( applyFilters: (query: any) => any, columns: string[], @@ -2822,8 +2887,9 @@ export class MemoryStore { /** * The locked body of update(). Callers must already hold the write lock - * and the serialized-update slot; transformMetadata composes it with a - * fresh read-decide step under the same lock. + * and the serialized-update slot: update() wraps it, transformMetadata + * composes it with a fresh read-decide step under the same lock, and the + * supersede commit path calls it from inside its own atomic section. */ private async performUpdateLocked( id: string, @@ -3172,6 +3238,103 @@ export class MemoryStore { return { repaired, failed, skipped, unrecovered }; } + + /** + * Force the open table handle onto the latest committed version. A nonzero + * readConsistencyInterval lets reads serve a snapshot up to that many + * seconds stale; a locked read-modify-write section must observe every + * commit that preceded its lock acquisition, so it re-syncs first. A sync + * failure propagates: proceeding on a possibly-stale snapshot would + * silently reintroduce the staleness this guard exists to close. + */ + private async syncTableToLatest(): Promise { + const table = this.table as unknown as { checkoutLatest?: () => Promise } | null; + if (table && typeof table.checkoutLatest === "function") { + await table.checkoutLatest(); + } + } + + /** + * Atomic supersede-and-store: re-discovers the target rows, inserts the new + * row, and invalidates every confirmed target inside ONE write-lock + + * serialized-update section. The caller's advisory discovery only decides + * whether to enter this path; the target set that actually commits is the + * one discovered here, so two concurrent same-key writers converge on a + * single active row (the second writer's recheck sees the first writer's + * replacement and supersedes it) instead of leaving both replacements + * standing. + * + * Only CONFIRMED invalidations are reported in supersededIds; a null or + * throwing patch lands in invalidationFailures instead of being silently + * counted as success. + */ + async storeSuperseding(options: { + entry: Omit; + discoverTargets: () => Promise; + finalizeEntryMetadata?: (targets: MemoryEntry[]) => string; + buildTargetPatch: (target: MemoryEntry, newEntryId: string) => MetadataPatch; + scopeFilter?: string[]; + }): Promise<{ + entry: MemoryEntry; + supersededIds: string[]; + invalidationFailures: Array<{ id: string; reason: string }>; + }> { + await this.ensureInitialized(); + const result = await this.runWithWriteLock(() => this.runSerializedUpdate(async () => { + // The cross-process lock serializes writers but does not refresh this + // handle's read snapshot: with a second store instance and a nonzero + // readConsistencyInterval, the locked re-discovery could miss the + // preceding writer's commit and leave both replacements active. + await this.syncTableToLatest(); + const targets = await options.discoverTargets(); + + const fullEntry: MemoryEntry = { + ...options.entry, + id: randomUUID(), + timestamp: Date.now(), + metadata: options.finalizeEntryMetadata + ? options.finalizeEntryMetadata(targets) + : options.entry.metadata || "{}", + importance: clampImportance(Number(options.entry.importance)), + } as MemoryEntry; + await this.table!.add([fullEntry]); + + const supersededIds: string[] = []; + const invalidationFailures: Array<{ id: string; reason: string }> = []; + for (const target of targets) { + try { + const existing = await this.getById(target.id, options.scopeFilter); + if (!existing) { + invalidationFailures.push({ + id: target.id, + reason: "row not found or outside accessible scopes at commit time", + }); + continue; + } + const metadata = buildSmartMetadata(existing, options.buildTargetPatch(existing, fullEntry.id)); + const updated = await this.performUpdateLocked( + target.id, + { metadata: stringifySmartMetadata(metadata) }, + options.scopeFilter, + ); + if (updated == null) { + invalidationFailures.push({ id: target.id, reason: "update persisted no row" }); + } else { + supersededIds.push(target.id); + } + } catch (err) { + invalidationFailures.push({ + id: target.id, + reason: err instanceof Error ? err.message : String(err), + }); + } + } + + return { entry: fullEntry, supersededIds, invalidationFailures }; + })); + this.noteDataModification(); + return result; + } async bulkDelete(scopeFilter: string[], beforeTimestamp?: number): Promise { await this.ensureInitialized(); diff --git a/src/tools.ts b/src/tools.ts index 93af825f..f4ab9092 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -74,6 +74,10 @@ interface ToolContext { mdMirror?: MdMirrorWriter | null; workspaceBoundary?: WorkspaceBoundaryConfig; selfImprovementMaxEntries?: number; + // Manual-priority store lane: a memory_store that finds a similar existing + // row supersedes it instead of being rejected as a duplicate; the manual + // text always lands verbatim. + manualStoreSupersede?: boolean; // Mirrors MemoryCliContext's onMemoriesDeleted (cli.ts): lets the host invalidate // in-process reflection caches after a live delete, not just CLI delete/delete-bulk. onMemoriesDeleted?: (info: { scopeFilter?: string[] }) => void; @@ -240,10 +244,22 @@ function hasExplicitMetadataField(entry: MemoryEntry, field: string): boolean { return Object.prototype.hasOwnProperty.call(parseMetadataObject(entry.metadata), field); } +/** + * The one normalization rule for comparing fact keys: explicit keys are stored + * trimmed but case-preserved while derived keys are lowercased, so every + * comparison site must apply the same trim+lowercase rule or a mixed-case + * explicit key ("Preferences:Theme") diverges from its query-equivalent + * derived form ("preferences:theme"). + */ +function normalizeFactKeyForComparison(key: string | undefined): string | undefined { + const normalized = key?.trim().toLowerCase(); + return normalized ? normalized : undefined; +} + function factQueryMatches(entry: MemoryEntry, query: string | undefined, factKey: string | undefined): boolean { const meta = parseSmartMetadata(entry.metadata, entry); - const normalizedFactKey = factKey?.trim().toLowerCase(); - if (normalizedFactKey && meta.fact_key?.toLowerCase() !== normalizedFactKey) return false; + const normalizedFactKey = normalizeFactKeyForComparison(factKey); + if (normalizedFactKey && normalizeFactKeyForComparison(meta.fact_key) !== normalizedFactKey) return false; const normalizedQuery = query?.trim().toLowerCase(); if (!normalizedQuery) return true; @@ -291,6 +307,76 @@ function serializeFactEntry(entry: MemoryEntry, atMs: number) { const FACT_QUERY_PAGE_SIZE = 500; +/** + * Hard ceiling on candidate rows a fact-key collision scan will accept from + * the store's bounded candidate query (see + * MemoryStore.listFactKeyCandidates). The scan runs inside + * storeSuperseding()'s cross-process write lock, so its work must stay + * bounded; a candidate set past this ceiling cannot guarantee a complete + * collision set, and an opted-in supersede write is REJECTED explicitly + * (FactKeyScanOverBoundError) rather than silently leaving the old same-key + * value active alongside the new one. + */ +const FACT_KEY_SCAN_MAX_ROWS = 20_000; + +/** + * An opted-in manual supersede write was rejected because the fact-key + * collision scan could not be completed within its bound. Callers surface + * this to the agent explicitly; `force: true` stores without superseding. + */ +export class FactKeyScanOverBoundError extends Error { + constructor(bound: number) { + super( + `manual supersede rejected: the scope holds more than ${bound} fact-key candidate rows, so a complete collision scan cannot be guaranteed. Retry with force: true to store without superseding, or reduce the scope's row count.`, + ); + this.name = "FactKeyScanOverBoundError"; + } +} + +/** + * Complete, scope-aware lookup of ACTIVE rows holding a fact key. The vector + * top-K is the wrong tool for key collisions: a same-key row ranked behind + * closer unrelated neighbors, or below the similarity floor, is exactly the + * stale value a manual update must supersede. The store performs a bounded, + * scope-only candidate query (hard limit push-down, no sort, no content + * narrowing: effective keys hide in any valid JSON layout or derive from the + * storage category column, so a serialization-layout pattern can exclude a + * valid candidate); exact normalized-key and active-row filtering happen + * here. An over-bound candidate set throws FactKeyScanOverBoundError — + * never a silently incomplete collision set. + */ +async function findActiveFactKeyEntries( + store: MemoryStore, + scopeFilter: string[], + factKey: string, +): Promise { + const matches: MemoryEntry[] = []; + const targetKey = normalizeFactKeyForComparison(factKey); + if (!targetKey) return matches; + const now = Date.now(); + const rows = await store.listFactKeyCandidates( + scopeFilter, + FACT_KEY_SCAN_MAX_ROWS, + ); + if (rows.length > FACT_KEY_SCAN_MAX_ROWS) { + throw new FactKeyScanOverBoundError(FACT_KEY_SCAN_MAX_ROWS); + } + for (const entry of rows) { + // Exact scope match: store reads mask legacy NULL scopes as "global" and + // store-level filters have passed legacy rows through before; a + // supersede scan must never treat a row outside the requested scopes + // as a collision target. + if (!scopeFilter.includes(entry.scope)) continue; + const meta = parseSmartMetadata(entry.metadata, entry); + if (!isMemoryActiveAt(meta, now) || isMemoryExpired(meta, now)) continue; + const entryKey = normalizeFactKeyForComparison( + meta.fact_key ?? deriveFactKey(meta.memory_category, entry.text), + ); + if (entryKey === targetKey) matches.push(entry); + } + return matches; +} + type FactQueryCandidate = { entry: MemoryEntry; meta: ReturnType; @@ -1341,19 +1427,122 @@ export function registerMemoryStoreTool( // Align with TEMPORAL_VERSIONED_CATEGORIES at the smart-category // layer so legacy storage categories like "fact" don't cross-match // unrelated profile/case memories. - let existing: Awaited> = []; + const manualSupersede = runtimeContext.manualStoreSupersede === true; + const newFactKey = deriveFactKey(memoryCategory, stripped); + + // One discovery routine serves both passes: the ADVISORY pass below + // decides whether to enter the supersede path at all, and the store's + // atomic commit re-runs it under the write lock so the target set + // that actually commits reflects concurrent writers' work. + const runSupersedeDiscovery = async (): Promise<{ + neighbors: Awaited>; + duplicateCandidate: Awaited>[number] | undefined; + manual: boolean; + targets: Array<{ entry: MemoryEntry; score?: number }>; + }> => { + // Check for duplicates / supersede candidates using raw vector + // similarity (bypasses importance/recency weighting). + // Fail-open by design: dedup must never block a legitimate write. + // excludeInactive: superseded historical records must not block + // new writes. + let neighbors: Awaited> = []; + try { + neighbors = await runtimeContext.store.vectorSearch(vector, 3, 0.1, [ + targetScope, + ], { excludeInactive: true }); + } catch (err) { + console.warn( + `memory-lancedb-pro: duplicate pre-check failed, continue store: ${String(err)}`, + ); + } + const duplicateCandidate = neighbors[0]?.score > 0.98 ? neighbors[0] : undefined; + // Key collisions resolve through a COMPLETE scope lookup, never the + // vector top-K: a same-key row ranked behind closer unrelated + // neighbors, or below the similarity floor, is exactly the stale + // value this path exists to supersede. Fail-open like the duplicate + // pre-check: a lookup error stores alongside instead of blocking. + let activeFactKeyEntries: MemoryEntry[] = []; + if (manualSupersede && !force && newFactKey) { + try { + activeFactKeyEntries = await findActiveFactKeyEntries( + runtimeContext.store, + [targetScope], + newFactKey, + ); + } catch (err) { + // An over-bound candidate set is NOT the fail-open class: a + // store-alongside here silently abandons the supersession + // invariant, so the write is rejected explicitly instead. + if (err instanceof FactKeyScanOverBoundError) { + throw err; + } + console.warn( + `memory-lancedb-pro: fact-key lookup failed, continue store: ${String(err)}`, + ); + } + } + const manualPriorityTargets: Array<{ entry: MemoryEntry; score?: number }> = []; + if (manualSupersede && !force) { + if (duplicateCandidate) { + manualPriorityTargets.push(duplicateCandidate); + } + for (const entry of activeFactKeyEntries) { + if (!manualPriorityTargets.some((target) => target.entry.id === entry.id)) { + manualPriorityTargets.push({ entry }); + } + } + } + // Auto-supersede band: similar memory (0.95-0.98), same + // storage-layer category, eligible category. + const bandCandidate = neighbors.find( + (r) => + r.score > 0.95 && + r.score <= 0.98 && + TEMPORAL_VERSIONED_CATEGORIES.has(memoryCategory) && + matchesMemoryCategoryFilter(r.entry.category, memoryCategory, r.entry.metadata), + ); + const targets = manualPriorityTargets.length > 0 + ? manualPriorityTargets + : bandCandidate + ? [bandCandidate] + : []; + return { + neighbors, + duplicateCandidate, + manual: manualPriorityTargets.length > 0, + targets, + }; + }; + + // An over-bound fact-key scan rejects the opted-in write explicitly: + // storing alongside would silently leave the old same-key value + // active, and pretending "created" hides exactly the failure the + // supersede contract exists to prevent. force: true bypasses the + // scan entirely and stores without superseding. + const buildOverBoundRejection = (err: FactKeyScanOverBoundError) => ({ + content: [ + { + type: "text", + text: err.message, + }, + ], + details: { + action: "rejected", + reason: "fact-key-scan-over-bound", + }, + }); + + let discovery: Awaited>; try { - existing = await runtimeContext.store.vectorSearch(vector, 3, 0.1, [ - targetScope, - ], { excludeInactive: true }); + discovery = await runSupersedeDiscovery(); } catch (err) { - console.warn( - `memory-lancedb-pro: duplicate pre-check failed, continue store: ${String(err)}`, - ); + if (err instanceof FactKeyScanOverBoundError) { + return buildOverBoundRejection(err); + } + throw err; } - - const duplicateCandidate = existing[0]?.score > 0.98 ? existing[0] : undefined; - if (duplicateCandidate && !force) { + const duplicateCandidate = discovery.duplicateCandidate; + if (duplicateCandidate && !force && !manualSupersede) { return { content: [ { @@ -1371,78 +1560,135 @@ export function registerMemoryStoreTool( }; } - // Auto-supersede: if a similar memory exists (0.95-0.98 similarity), - // same storage-layer category, and category is eligible, mark the old - // one as superseded and store the new one with a supersedes link. - const supersedeCandidate = existing.find( - (r) => - r.score > 0.95 && - r.score <= 0.98 && - TEMPORAL_VERSIONED_CATEGORIES.has(memoryCategory) && - matchesMemoryCategoryFilter(r.entry.category, memoryCategory, r.entry.metadata), - ); + // Manual-priority supersede (manualStoreSupersede): a manual store + // always takes priority — its text lands verbatim, and a similar + // existing row yields to it. Targets, in order: the near-identical + // neighbor the duplicate check used to reject, then an active + // neighbor holding the same fact key at any similarity (the + // update/contradiction shape, e.g. a new value for a versioned + // fact). Anything else falls through to the versioned-band check, + // and past that stores alongside: a wrong supersede destroys a real + // fact, while a duplicate is fixable noise. + // A manual-priority write with an EMPTY advisory still enters the + // locked path: two first-time same-key writers otherwise both see + // nothing and both plain-store, leaving two active rows for one + // fact key. The locked rediscovery is authoritative; when it also + // finds nothing, the commit is a plain create. + const manualPriorityWrite = manualSupersede && !force; + if (discovery.targets.length > 0 || manualPriorityWrite) { + // Canonical identity comes from the REQUESTED store, never from a + // near-duplicate donor: the new row's category and fact key are the + // requested ones, and overview/tier inherit only from a + // category-verified target (the band shape). Temporal expiry is + // preserved exactly like the plain-store path. + const buildSupersedeMetadata = (targets: MemoryEntry[], manualPriority: boolean): string => { + const now = Date.now(); + const verified = targets + .map((target) => ({ target, meta: parseSmartMetadata(target.metadata, target) })) + .find(({ target }) => + matchesMemoryCategoryFilter(target.category, memoryCategory, target.metadata), + ); + // The band keeps the verified target's ESTABLISHED key: deriving + // one from the replacement's wording would split the fact's + // history across two canonical identities. Requested-key + // precedence is the opt-in manual-priority contract only. + const factKey = manualPriority + ? newFactKey ?? verified?.meta.fact_key ?? undefined + : verified?.meta.fact_key ?? newFactKey ?? undefined; + const primary = targets[0]; + return stringifySmartMetadata(buildSmartMetadata( + { text, category: storageCategory, importance: safeImportance }, + { + l0_abstract: text, + // A manual-priority supersede replaces the fact's VALUE, so + // the old row's overview is stale by definition; the band + // case keeps the richer overview of its category-verified + // target as before. + l1_overview: manualPriority ? `- ${text}` : verified?.meta.l1_overview || `- ${text}`, + l2_content: text, + memory_category: memoryCategory, + tier: verified?.meta.tier, + source: "manual", + state: "confirmed", + memory_layer: deriveManualMemoryLayer(memoryCategory), + last_confirmed_use_at: now, + bad_recall_count: 0, + suppressed_until_turn: 0, + valid_from: now, + memory_temporal_type: temporalType, + valid_until: validUntil, + ...(factKey ? { fact_key: factKey } : {}), + ...(primary ? { supersedes: primary.id } : {}), + relations: targets.reduce( + (relations, target) => + appendRelation(relations, { + type: "supersedes", + targetId: target.id, + }), + [] as ReturnType, + ), + }, + )); + }; - if (supersedeCandidate) { - const oldEntry = supersedeCandidate.entry; - const oldMeta = parseSmartMetadata(oldEntry.metadata, oldEntry); - const now = Date.now(); - const factKey = - oldMeta.fact_key ?? deriveFactKey(oldMeta.memory_category, text); - - // Store new memory with supersedes link, preserving canonical fields - // from the old entry (aligns with memory_update supersede path). - const newMeta = buildSmartMetadata( - { text, category: storageCategory, importance: safeImportance }, - { - l0_abstract: text, - l1_overview: oldMeta.l1_overview || `- ${text}`, - l2_content: text, - memory_category: oldMeta.memory_category, - tier: oldMeta.tier, - source: "manual", - state: "confirmed", - memory_layer: deriveManualMemoryLayer(oldMeta.memory_category), - last_confirmed_use_at: now, - bad_recall_count: 0, - suppressed_until_turn: 0, - valid_from: now, - fact_key: factKey, - supersedes: oldEntry.id, - relations: appendRelation([], { - type: "supersedes", - targetId: oldEntry.id, + const buildInvalidationPatch = (target: MemoryEntry, newEntryId: string) => { + const targetMeta = parseSmartMetadata(target.metadata, target); + const sameCategory = matchesMemoryCategoryFilter(target.category, memoryCategory, target.metadata); + return { + // Backfill a missing fact key only on a category-verified + // target: stamping the new key onto a foreign-category + // near-duplicate would misfile it. + ...(!targetMeta.fact_key && sameCategory && newFactKey ? { fact_key: newFactKey } : {}), + invalidated_at: Date.now(), + superseded_by: newEntryId, + relations: appendRelation(targetMeta.relations, { + type: "superseded_by", + targetId: newEntryId, }), - }, - ); - - const newEntry = await runtimeContext.store.store({ - text, - vector, - importance: safeImportance, - category: storageCategory, - scope: targetScope, - metadata: stringifySmartMetadata(newMeta), - }); + }; + }; - // Invalidate old record + // Commit atomically at the store layer: recheck, insert, and + // invalidate run under one write lock, so concurrent same-key + // writers converge on a single active row instead of leaving two + // replacements standing. + let lastDiscovery = discovery; + let committed: Awaited>; try { - await runtimeContext.store.patchMetadata( - oldEntry.id, - { - fact_key: factKey, - invalidated_at: now, - superseded_by: newEntry.id, - relations: appendRelation(oldMeta.relations, { - type: "superseded_by", - targetId: newEntry.id, - }), + committed = await runtimeContext.store.storeSuperseding({ + entry: { + text, + vector, + importance: safeImportance, + category: storageCategory, + scope: targetScope, + metadata: "{}", }, - [targetScope], - ); - } catch (patchErr) { - // New record is already the source of truth; log but don't fail + scopeFilter: [targetScope], + discoverTargets: async () => { + lastDiscovery = await runSupersedeDiscovery(); + return lastDiscovery.targets.map((target) => target.entry); + }, + finalizeEntryMetadata: (targets) => + buildSupersedeMetadata(targets, lastDiscovery.manual), + buildTargetPatch: buildInvalidationPatch, + }); + } catch (err) { + // The locked recheck hit the scan bound (the scope crossed it + // after the advisory pass): the write aborts before the insert, + // so rejecting here leaves no partial state behind. + if (err instanceof FactKeyScanOverBoundError) { + return buildOverBoundRejection(err); + } + throw err; + } + const newEntry = committed.entry; + const { supersededIds, invalidationFailures } = committed; + for (const failure of invalidationFailures) { + // The new record is already stored; surface the unconfirmed + // invalidation instead of silently reporting it as superseded. console.warn( - `memory-pro: failed to patch superseded record ${oldEntry.id.slice(0, 8)}: ${patchErr}`, + `memory-pro: failed to invalidate superseded record ${failure.id.slice(0, 8)}: ${failure.reason}`, ); } @@ -1454,22 +1700,54 @@ export function registerMemoryStoreTool( ); } + if (lastDiscovery.targets.length === 0) { + // First writer of this fact: the authoritative locked discovery + // found nothing to supersede, so this commit was a plain create + // and must report as one, not as "superseded no memories". + return { + content: [ + { + type: "text", + text: `Stored: "${text.slice(0, 100)}${text.length > 100 ? "..." : ""}" in scope '${targetScope}'`, + }, + ], + details: { + action: "created", + id: newEntry.id, + scope: newEntry.scope, + category: memoryCategory, + rawCategory: newEntry.category, + importance: newEntry.importance, + }, + }; + } + + const supersededLabel = supersededIds.length > 1 + ? `${supersededIds.length} memories (${supersededIds.map((id) => id.slice(0, 8)).join(", ")})` + : supersededIds.length === 1 + ? `memory ${supersededIds[0].slice(0, 8)}...` + : "no memories"; + const failureSuffix = invalidationFailures.length > 0 + ? ` (${invalidationFailures.length} invalidation(s) failed; those rows may still be active)` + : ""; return { content: [ { type: "text", - text: `Superseded memory ${oldEntry.id.slice(0, 8)}... → new version ${newEntry.id.slice(0, 8)}...: "${text.slice(0, 80)}${text.length > 80 ? "..." : ""}"`, + text: `Superseded ${supersededLabel} → new version ${newEntry.id.slice(0, 8)}...: "${text.slice(0, 80)}${text.length > 80 ? "..." : ""}"${failureSuffix}`, }, ], details: { action: "superseded", id: newEntry.id, - supersededId: oldEntry.id, + supersededId: supersededIds[0] ?? null, + supersededIds, + ...(invalidationFailures.length > 0 ? { invalidationFailures } : {}), scope: newEntry.scope, category: memoryCategory, rawCategory: newEntry.category, importance: newEntry.importance, - similarity: supersedeCandidate.score, + similarity: lastDiscovery.targets[0]?.score, }, }; } diff --git a/test/is-latest-auto-supersede.test.mjs b/test/is-latest-auto-supersede.test.mjs index d18ee59a..c56f3a1d 100644 --- a/test/is-latest-auto-supersede.test.mjs +++ b/test/is-latest-auto-supersede.test.mjs @@ -79,6 +79,28 @@ function makeMockStore() { const patched = buildSmartMetadata(entry, patch); entry.metadata = stringifySmartMetadata(patched); }, + // In-memory mirror of MemoryStore.storeSuperseding: re-runs discovery, + // stores the new row, patches each target in place, reports confirmed ids. + async storeSuperseding({ entry, discoverTargets, finalizeEntryMetadata, buildTargetPatch }) { + const targets = await discoverTargets(); + const stored = await this.store({ + ...entry, + metadata: finalizeEntryMetadata ? finalizeEntryMetadata(targets) : entry.metadata, + }); + const supersededIds = []; + const invalidationFailures = []; + for (const target of targets) { + const existing = entries.get(target.id); + if (!existing) { + invalidationFailures.push({ id: target.id, reason: "row not found" }); + continue; + } + const patched = buildSmartMetadata(existing, buildTargetPatch(existing, stored.id)); + existing.metadata = stringifySmartMetadata(patched); + supersededIds.push(target.id); + } + return { entry: stored, supersededIds, invalidationFailures }; + }, hasFtsSupport: false, }; } diff --git a/test/manual-store-supersede.test.mjs b/test/manual-store-supersede.test.mjs new file mode 100644 index 00000000..f89f87e1 --- /dev/null +++ b/test/manual-store-supersede.test.mjs @@ -0,0 +1,998 @@ +/** + * Manual memory_store lane: always-store supersede semantics. + * + * Design ruling: a manual store ALWAYS takes priority. It skips the admission + * judge, and dedup treats it "in a different way": when a similar memory + * exists, the OLD row is superseded/invalidated by the manual one, and the + * manual text is ALWAYS stored verbatim, never mutated and never dropped. + * + * Supersede triggers with `manualStoreSupersede: true` (deterministic, no + * LLM on this lane): + * 1. near-identical neighbor (similarity > 0.98) — previously a reject; + * 2. fact-key collision with an active neighbor at any similarity — the + * contradiction/update shape ("favorite drink: tea" vs the fizzwick row); + * 3. the existing 0.95-0.98 same-category versioned band (unchanged). + * Anything else creates alongside: a wrong supersede destroys a real fact, + * a duplicate is fixable noise, so the gray zone stays on the safe side. + * + * With the knob off (upstream default) the lane behaves exactly as before, + * including the > 0.98 duplicate reject. + * + * Fixtures are entirely synthetic; no real fleet data. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { registerAllMemoryTools } = jiti("../src/tools.ts"); +const { MemoryStore } = jiti("../src/store.ts"); +const { parseSmartMetadata, isMemoryActiveAt, deriveFactKey } = jiti("../src/smart-metadata.ts"); +const { classifyTemporal, inferExpiry } = jiti("../src/temporal-classifier.ts"); + +function createToolSet(context) { + const creators = new Map(); + const api = { + registerTool(factory, meta) { + creators.set(meta.name, factory); + }, + logger: { info() {}, warn() {}, debug() {} }, + }; + registerAllMemoryTools(api, context, { enableManagementTools: true }); + return { + get(name) { + const factory = creators.get(name); + assert.ok(factory, `tool ${name} should be registered`); + return factory({}); + }, + }; +} + +function neighborRow({ id = "old-1", text, category = "preference", score, factKey, memoryCategory = "preferences" }) { + return { + entry: { + id, + text, + category, + scope: "agent:main", + importance: 0.7, + timestamp: Date.now() - 60_000, + metadata: JSON.stringify({ + memory_category: memoryCategory, + l0_abstract: text, + l1_overview: `- ${text}`, + l2_content: text, + source: "auto-capture", + state: "confirmed", + ...(factKey ? { fact_key: factKey } : {}), + }), + }, + score, + }; +} + +function makeContext({ neighbors = [], rows, manualStoreSupersede, patchBehavior } = {}) { + const storedEntries = []; + const patchCalls = []; + // Production-shaped store double: vectorSearch honors the caller's limit and + // minScore exactly like the real store, and list() pages over ALL rows the + // store holds (`rows` defaults to the vector neighbors' entries). + const allRows = rows ?? neighbors.map((neighbor) => neighbor.entry); + const context = { + agentId: "main", + workspaceDir: "/tmp", + mdMirror: null, + ...(manualStoreSupersede === undefined ? {} : { manualStoreSupersede }), + scopeManager: { + getAccessibleScopes: (agentId) => ["global", `agent:${agentId}`], + getScopeFilter: (agentId) => ["global", `agent:${agentId}`], + isAccessible: (scope, agentId) => ["global", `agent:${agentId}`].includes(scope), + getDefaultScope: (agentId) => `agent:${agentId}`, + }, + retriever: { + getConfig() { + return { mode: "hybrid" }; + }, + }, + store: { + async vectorSearch(vector, limit = 5, minScore = 0.3) { + return neighbors + .filter((neighbor) => neighbor.score >= minScore) + .sort((a, b) => b.score - a.score) + .slice(0, limit); + }, + async list(scopeFilter, category, limit = 100, offset = 0) { + return allRows.slice(offset, offset + limit); + }, + // Production-shaped double of MemoryStore.listFactKeyCandidates: + // scope-only narrowing (effective keys hide in any valid JSON layout or + // derive from the storage category column, so content patterns are + // unsound) with the bound+1 return contract. + async listFactKeyCandidates(scopeFilter, bound) { + const candidates = allRows.filter((row) => { + if (scopeFilter && scopeFilter.length > 0 && !scopeFilter.includes(row.scope)) return false; + return true; + }); + return candidates.slice(0, bound + 1); + }, + async store(entry) { + const stored = { ...entry, id: `new-${storedEntries.length + 1}`, timestamp: Date.now() }; + storedEntries.push(stored); + return stored; + }, + async patchMetadata(id, patch, scopeFilter) { + patchCalls.push({ id, patch, scopeFilter }); + return null; + }, + // Production-shaped double of MemoryStore.storeSuperseding: re-runs the + // caller's discovery, stores the new row, applies each target patch, and + // reports CONFIRMED invalidations only. patchBehavior lets tests model + // null returns, throws, and partial success. + async storeSuperseding({ entry, discoverTargets, finalizeEntryMetadata, buildTargetPatch, scopeFilter }) { + const targets = await discoverTargets(); + const stored = { ...entry, id: `new-${storedEntries.length + 1}`, timestamp: Date.now() }; + if (finalizeEntryMetadata) { + stored.metadata = finalizeEntryMetadata(targets); + } + storedEntries.push(stored); + const supersededIds = []; + const invalidationFailures = []; + for (const target of targets) { + try { + const patch = buildTargetPatch(target, stored.id); + patchCalls.push({ id: target.id, patch, scopeFilter }); + const outcome = patchBehavior ? await patchBehavior(target, patch) : { ...target }; + if (outcome == null) { + invalidationFailures.push({ id: target.id, reason: "update persisted no row" }); + } else { + supersededIds.push(target.id); + } + } catch (err) { + invalidationFailures.push({ id: target.id, reason: err instanceof Error ? err.message : String(err) }); + } + } + return { entry: stored, supersededIds, invalidationFailures }; + }, + }, + embedder: { + async embedPassage() { + return [0.1, 0.2, 0.3]; + }, + }, + }; + return { context, storedEntries, patchCalls }; +} + +describe("manual memory_store always-store supersede semantics", () => { + it("supersedes instead of rejecting when a near-identical memory exists (knob on)", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [neighborRow({ text: "favorite drink: Fizzwick", score: 0.99, factKey: "preferences:favorite drink" })], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const input = "favorite drink: Fizzwick Zero"; + const res = await store.execute(null, { text: input, category: "preference" }); + + assert.equal(res.details.action, "superseded", "a near-identical manual store must land as a supersede, never a reject"); + assert.equal(storedEntries.length, 1, "the manual row must always be stored"); + assert.equal(storedEntries[0].text, input, "the manual text must be stored verbatim, never mutated"); + assert.equal(patchCalls.length, 1, "the old row must be invalidated"); + assert.equal(patchCalls[0].id, "old-1"); + assert.ok(patchCalls[0].patch.invalidated_at > 0); + assert.equal(patchCalls[0].patch.superseded_by, storedEntries[0].id ?? "new-1"); + }); + + it("supersedes on a fact-key collision even at low vector similarity (contradiction shape, knob on)", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [neighborRow({ text: "favorite drink: Fizzwick", score: 0.8, factKey: "preferences:favorite drink" })], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const input = "favorite drink: tea"; + const res = await store.execute(null, { text: input, category: "preference" }); + + assert.equal(res.details.action, "superseded", "a same-fact-key update must supersede the old value"); + assert.equal(storedEntries.length, 1); + assert.equal(storedEntries[0].text, input, "the manual text must be stored verbatim"); + assert.equal(patchCalls.length, 1); + assert.equal(patchCalls[0].id, "old-1"); + }); + + it("creates alongside when the neighbor is similar but a different fact (knob on)", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [neighborRow({ text: "favorite food: lahmacun", score: 0.85, factKey: "preferences:favorite food" })], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const input = "favorite drink: tea"; + const res = await store.execute(null, { text: input, category: "preference" }); + + assert.equal(res.details.action, "created", "a different fact must not be invalidated, however vector-close"); + assert.equal(storedEntries.length, 1); + assert.equal(storedEntries[0].text, input); + assert.equal(patchCalls.length, 0, "no supersede may fire for an unrelated fact"); + }); + + it("force still bypasses the supersede path entirely (knob on)", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [neighborRow({ text: "favorite drink: Fizzwick", score: 0.99, factKey: "preferences:favorite drink" })], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: Fizzwick", category: "preference", force: true }); + + assert.equal(res.details.action, "created", "force stores alongside without touching the old row"); + assert.equal(storedEntries.length, 1); + assert.equal(patchCalls.length, 0); + }); + + it("keeps the upstream duplicate reject when the knob is off (compat default)", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + neighbors: [neighborRow({ text: "favorite drink: Fizzwick", score: 0.99, factKey: "preferences:favorite drink" })], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: Fizzwick", category: "preference" }); + + assert.equal(res.details.action, "duplicate", "knob off must preserve the upstream duplicate check exactly"); + assert.equal(storedEntries.length, 0); + assert.equal(patchCalls.length, 0); + }); + + it("supersedes a same-key row the vector top-K cannot see (ranked behind three closer unrelated neighbors)", async () => { + const staleKeyRow = neighborRow({ id: "old-key", text: "favorite drink: Fizzwick", score: 0.5, factKey: "preferences:favorite drink" }); + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [ + neighborRow({ id: "near-1", text: "favorite snack: simit", score: 0.92, factKey: "preferences:favorite snack" }), + neighborRow({ id: "near-2", text: "favorite dessert: baklava", score: 0.91, factKey: "preferences:favorite dessert" }), + neighborRow({ id: "near-3", text: "favorite fruit: fig", score: 0.9, factKey: "preferences:favorite fruit" }), + staleKeyRow, + ], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: tea", category: "preference" }); + + assert.equal(res.details.action, "superseded", "the stale same-key row must be found even when it ranks fourth"); + assert.equal(patchCalls.length, 1, "only the same-key row may be invalidated"); + assert.equal(patchCalls[0].id, "old-key"); + assert.equal(storedEntries.length, 1); + }); + + it("supersedes a same-key row that falls below the vector similarity floor", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [neighborRow({ id: "old-faint", text: "favorite drink: Fizzwick", score: 0.05, factKey: "preferences:favorite drink" })], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: tea", category: "preference" }); + + assert.equal(res.details.action, "superseded", "the similarity floor must not hide a same-key collision"); + assert.equal(patchCalls.length, 1); + assert.equal(patchCalls[0].id, "old-faint"); + assert.equal(storedEntries.length, 1); + }); + + it("supersedes EVERY active same-key row, not just the first match", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [ + neighborRow({ id: "old-a", text: "favorite drink: Fizzwick", score: 0.6, factKey: "preferences:favorite drink" }), + neighborRow({ id: "old-b", text: "favorite drink: ayran", score: 0.55, factKey: "preferences:favorite drink" }), + ], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: tea", category: "preference" }); + + assert.equal(res.details.action, "superseded"); + assert.deepEqual([...res.details.supersededIds].sort(), ["old-a", "old-b"], "every active same-key row must be reconciled"); + assert.equal(patchCalls.length, 2, "both stale rows must be invalidated"); + assert.deepEqual(patchCalls.map((call) => call.id).sort(), ["old-a", "old-b"]); + assert.equal(storedEntries.length, 1, "exactly one new row carries the manual value"); + }); + + it("ignores already-invalidated same-key rows (history must not be re-superseded)", async () => { + const invalidated = neighborRow({ id: "old-history", text: "favorite drink: salep", score: 0.6, factKey: "preferences:favorite drink" }); + const meta = JSON.parse(invalidated.entry.metadata); + meta.invalidated_at = Date.now() - 1_000; + invalidated.entry.metadata = JSON.stringify(meta); + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [invalidated], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: tea", category: "preference" }); + + assert.equal(res.details.action, "created", "a historical superseded row is not an active collision"); + assert.equal(patchCalls.length, 0); + assert.equal(storedEntries.length, 1); + }); + + it("keeps the existing 0.95-0.98 same-category band superseding with the knob on (no regression)", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [neighborRow({ text: "favorite drink is Fizzwick for sure", score: 0.96 })], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const input = "favorite drink: Fizzwick Zero"; + const res = await store.execute(null, { text: input, category: "preference" }); + + assert.equal(res.details.action, "superseded"); + assert.equal(storedEntries.length, 1); + assert.equal(storedEntries[0].text, input); + assert.equal(patchCalls.length, 1); + }); + + it("reports only CONFIRMED invalidations: a null patch outcome is a failure, not a superseded id", async () => { + const { context, storedEntries } = makeContext({ + manualStoreSupersede: true, + neighbors: [ + neighborRow({ id: "old-a", text: "favorite drink: Fizzwick", score: 0.6, factKey: "preferences:favorite drink" }), + neighborRow({ id: "old-b", text: "favorite drink: ayran", score: 0.55, factKey: "preferences:favorite drink" }), + ], + patchBehavior: async (target) => (target.id === "old-b" ? null : { ...target }), + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: tea", category: "preference" }); + + assert.equal(res.details.action, "superseded"); + assert.equal(storedEntries.length, 1, "the manual row always lands"); + assert.deepEqual(res.details.supersededIds, ["old-a"], "only the confirmed invalidation may be reported"); + assert.equal(res.details.invalidationFailures.length, 1); + assert.equal(res.details.invalidationFailures[0].id, "old-b"); + assert.match(res.details.invalidationFailures[0].reason, /persisted no row/); + assert.match(res.content[0].text, /1 invalidation\(s\) failed/); + }); + + it("reports a thrown patch as a failure and keeps supersededId null when nothing is confirmed", async () => { + const { context, storedEntries } = makeContext({ + manualStoreSupersede: true, + neighbors: [neighborRow({ text: "favorite drink: Fizzwick", score: 0.99, factKey: "preferences:favorite drink" })], + patchBehavior: async () => { + throw new Error("synthetic patch failure"); + }, + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: tea", category: "preference" }); + + assert.equal(res.details.action, "superseded"); + assert.equal(storedEntries.length, 1, "the manual row always lands"); + assert.deepEqual(res.details.supersededIds, [], "an unconfirmed invalidation must not be reported as superseded"); + assert.equal(res.details.supersededId, null); + assert.equal(res.details.invalidationFailures.length, 1); + assert.match(res.details.invalidationFailures[0].reason, /synthetic patch failure/); + }); + + it("builds canonical metadata from the REQUESTED category and fact key, not a foreign-category near-duplicate", async () => { + const foreignDonor = neighborRow({ + id: "foreign-1", + text: "favorite drink: tea ceremony is my hobby", + category: "entity", + memoryCategory: "profile", + score: 0.99, + factKey: "profile:owner hobby", + }); + const donorMeta = JSON.parse(foreignDonor.entry.metadata); + donorMeta.tier = "core"; + foreignDonor.entry.metadata = JSON.stringify(donorMeta); + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [foreignDonor], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const input = "favorite drink: tea"; + const res = await store.execute(null, { text: input, category: "preference" }); + + assert.equal(res.details.action, "superseded", "the near-identical row is still superseded"); + const meta = JSON.parse(storedEntries[0].metadata); + assert.equal(meta.memory_category, "preferences", "the requested category is canonical"); + assert.equal(meta.fact_key, deriveFactKey("preferences", input), "the NEW fact key is canonical, never the donor's"); + assert.notEqual(meta.fact_key, "profile:owner hobby"); + assert.notEqual(meta.tier, "core", "tier must not be inherited from a foreign-category donor"); + assert.equal(meta.memory_temporal_type, classifyTemporal(input), "temporal classification must survive the supersede branch"); + assert.equal(meta.valid_until, inferExpiry(input), "temporal expiry must survive the supersede branch"); + assert.equal(patchCalls.length, 1); + assert.equal( + patchCalls[0].patch.fact_key, + undefined, + "a foreign-category target must not be backfilled with the new fact key", + ); + }); + + it("preserves the verified target's explicit fact key on the default band (knob off)", async () => { + // The 0.95-0.98 band is the pre-existing default path. Its canonical + // identity is the ESTABLISHED key on the verified target: deriving a + // fresh key from the replacement's wording splits the fact's history + // across two identities, so later fact-key updates miss the replacement. + // Requested-key precedence belongs to the opt-in manual-priority lane. + const target = neighborRow({ + text: "User prefers the synthetic dark theme in every editor.", + score: 0.96, + factKey: "preferences:theme", + }); + const { context, storedEntries } = makeContext({ + manualStoreSupersede: false, + neighbors: [target], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const input = "i prefer the synthetic light mode now"; + const res = await store.execute(null, { text: input, category: "preference" }); + + assert.equal(res.details.action, "superseded", "the band still supersedes with the knob off"); + const meta = JSON.parse(storedEntries[0].metadata); + assert.equal( + meta.fact_key, + "preferences:theme", + "the verified target's established key is the canonical identity on the default band", + ); + assert.notEqual(meta.fact_key, deriveFactKey("preferences", input)); + }); + + it("continues a keyless target's derived identity on the default band (knob off)", async () => { + // A target without an explicit key still HAS an identity: the metadata + // parser derives one from the target's own abstract. The replacement + // inherits that, so both rows keep resolving to the same key; the + // replacement's wording never mints the identity on the default band. + const targetText = "User prefers the synthetic dark theme in every editor."; + const target = neighborRow({ text: targetText, score: 0.96 }); + const { context, storedEntries } = makeContext({ + manualStoreSupersede: false, + neighbors: [target], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const input = "i prefer the synthetic light mode now"; + const res = await store.execute(null, { text: input, category: "preference" }); + + assert.equal(res.details.action, "superseded"); + const meta = JSON.parse(storedEntries[0].metadata); + assert.equal( + meta.fact_key, + deriveFactKey("preferences", targetText), + "the keyless target's own derived identity carries over to the replacement", + ); + assert.notEqual(meta.fact_key, deriveFactKey("preferences", input)); + }); +}); + +describe("manual supersede commits atomically at the store layer (real store)", () => { + function makeRealContext(dir) { + const store = new MemoryStore({ dbPath: dir, vectorDim: 3 }); + const context = { + agentId: "main", + workspaceDir: "/tmp", + mdMirror: null, + manualStoreSupersede: true, + scopeManager: { + getAccessibleScopes: (agentId) => ["global", `agent:${agentId}`], + getScopeFilter: (agentId) => ["global", `agent:${agentId}`], + isAccessible: (scope, agentId) => ["global", `agent:${agentId}`].includes(scope), + getDefaultScope: (agentId) => `agent:${agentId}`, + }, + retriever: { + getConfig() { + return { mode: "hybrid" }; + }, + }, + store, + embedder: { + // Every "favorite drink" text embeds identically, so concurrent writers + // see each other's rows as near-identical same-key neighbors. + async embedPassage() { + return [1, 0, 0]; + }, + }, + }; + return { context, store }; + } + + it("two concurrent same-key writers leave exactly ONE active row (locked recheck supersedes the earlier replacement)", async () => { + const dir = mkdtempSync(join(tmpdir(), "supersede-atomic-")); + const { context, store } = makeRealContext(dir); + try { + const factKey = deriveFactKey("preferences", "favorite drink: cola"); + await store.store({ + text: "favorite drink: cola", + vector: [1, 0, 0], + category: "preference", + scope: "agent:main", + importance: 0.7, + metadata: JSON.stringify({ + memory_category: "preferences", + fact_key: factKey, + source: "manual", + state: "confirmed", + l0_abstract: "favorite drink: cola", + }), + }); + + // Barrier: both writers finish their ADVISORY discovery before either + // commits, forcing the interleaving the lock must survive. Later calls + // (the locked rechecks) pass through freely. + const realVectorSearch = store.vectorSearch.bind(store); + let arrivals = 0; + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + store.vectorSearch = async (...args) => { + arrivals += 1; + if (arrivals <= 2) { + if (arrivals === 2) release(); + await gate; + } + return realVectorSearch(...args); + }; + + const tools = createToolSet(context); + const storeTool = tools.get("memory_store"); + const [resA, resB] = await Promise.all([ + storeTool.execute(null, { text: "favorite drink: tea", category: "preference" }), + storeTool.execute(null, { text: "favorite drink: coffee", category: "preference" }), + ]); + store.vectorSearch = realVectorSearch; + + assert.equal(resA.details.action, "superseded"); + assert.equal(resB.details.action, "superseded"); + + const rows = await store.list(undefined, undefined, 100, 0); + const now = Date.now(); + const activeSameKey = rows.filter((row) => { + const meta = parseSmartMetadata(row.metadata, row); + const key = meta.fact_key ?? deriveFactKey(meta.memory_category, row.text); + return key === factKey && isMemoryActiveAt(meta, now); + }); + assert.equal( + activeSameKey.length, + 1, + `exactly one active row may hold the fact key after concurrent writers (got: ${activeSameKey.map((row) => row.text).join(" | ")})`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("two concurrent FIRST writers of a fact key leave exactly ONE active row (empty advisory still enters the locked path)", async () => { + // The reviewer-probed gap: with NO pre-existing row, both writers' + // unlocked advisory discovery comes back empty, and a path that only + // enters the locked operation on a non-empty advisory lets both fall + // through to plain store(), leaving two active rows for one fact key. + // The locked rediscovery is authoritative: the first writer creates, the + // second must see that row under the lock and supersede it. + const dir = mkdtempSync(join(tmpdir(), "supersede-firstwrite-")); + const { context, store } = makeRealContext(dir); + try { + const factKey = deriveFactKey("preferences", "favorite drink: tea"); + + // Barrier: both writers finish their ADVISORY discovery before either + // commits. Later calls (the locked rechecks) pass through freely. + const realVectorSearch = store.vectorSearch.bind(store); + let arrivals = 0; + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + store.vectorSearch = async (...args) => { + arrivals += 1; + if (arrivals <= 2) { + if (arrivals === 2) release(); + await gate; + } + return realVectorSearch(...args); + }; + + const tools = createToolSet(context); + const storeTool = tools.get("memory_store"); + const [resA, resB] = await Promise.all([ + storeTool.execute(null, { text: "favorite drink: tea", category: "preference" }), + storeTool.execute(null, { text: "favorite drink: coffee", category: "preference" }), + ]); + store.vectorSearch = realVectorSearch; + + const actions = [resA.details.action, resB.details.action].sort(); + assert.deepEqual( + actions, + ["created", "superseded"], + "one writer creates, the other's locked recheck must supersede that row", + ); + + const rows = await store.list(undefined, undefined, 100, 0); + const now = Date.now(); + const activeSameKey = rows.filter((row) => { + const meta = parseSmartMetadata(row.metadata, row); + const key = meta.fact_key ?? deriveFactKey(meta.memory_category, row.text); + return key === factKey && isMemoryActiveAt(meta, now); + }); + assert.equal( + activeSameKey.length, + 1, + `exactly one active row may hold the fact key after concurrent first writers (got: ${activeSameKey.map((row) => row.text).join(" | ")})`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("a second store instance with a stale read snapshot still supersedes the first instance's replacement (nonzero readConsistencyInterval)", async () => { + const dir = mkdtempSync(join(tmpdir(), "supersede-xinst-")); + const makeInstance = () => { + const store = new MemoryStore({ dbPath: dir, vectorDim: 3, readConsistencyInterval: 30 }); + const context = { + agentId: "main", + workspaceDir: "/tmp", + mdMirror: null, + manualStoreSupersede: true, + scopeManager: { + getAccessibleScopes: (agentId) => ["global", `agent:${agentId}`], + getScopeFilter: (agentId) => ["global", `agent:${agentId}`], + isAccessible: (scope, agentId) => ["global", `agent:${agentId}`].includes(scope), + getDefaultScope: (agentId) => `agent:${agentId}`, + }, + retriever: { + getConfig() { + return { mode: "hybrid" }; + }, + }, + store, + embedder: { + async embedPassage() { + return [1, 0, 0]; + }, + }, + }; + return { context, store }; + }; + const first = makeInstance(); + const second = makeInstance(); + try { + const factKey = deriveFactKey("preferences", "favorite drink: cola"); + await first.store.store({ + text: "favorite drink: cola", + vector: [1, 0, 0], + category: "preference", + scope: "agent:main", + importance: 0.7, + metadata: JSON.stringify({ + memory_category: "preferences", + fact_key: factKey, + source: "manual", + state: "confirmed", + l0_abstract: "favorite drink: cola", + }), + }); + + // Arm the second instance's table snapshot BEFORE the first writer's + // supersede commits: with a 30s consistency interval this handle keeps + // serving that snapshot, so its locked recheck reads stale unless the + // store re-syncs the handle under the lock. + await second.store.list(undefined, undefined, 10, 0); + + const toolA = createToolSet(first.context).get("memory_store"); + const toolB = createToolSet(second.context).get("memory_store"); + + const resA = await toolA.execute(null, { text: "favorite drink: tea", category: "preference" }); + assert.equal(resA.details.action, "superseded"); + + const resB = await toolB.execute(null, { text: "favorite drink: coffee", category: "preference" }); + assert.equal(resB.details.action, "superseded"); + + const verifyStore = new MemoryStore({ dbPath: dir, vectorDim: 3 }); + const rows = await verifyStore.list(undefined, undefined, 100, 0); + const now = Date.now(); + const activeSameKey = rows.filter((row) => { + const meta = parseSmartMetadata(row.metadata, row); + const key = meta.fact_key ?? deriveFactKey(meta.memory_category, row.text); + return key === factKey && isMemoryActiveAt(meta, now); + }); + assert.equal( + activeSameKey.length, + 1, + `the serialized writers must converge on one active row even across stale instance snapshots (got: ${activeSameKey.map((row) => row.text).join(" | ")})`, + ); + assert.equal(activeSameKey[0].text, "favorite drink: coffee", "the last writer's replacement must be the surviving active row"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("supersedes a legacy empty-metadata row whose key derives from the storage category column (end to end)", async () => { + // A row with category="preference" and metadata="{}" carries no fact_key + // and no memory_category field, yet parseSmartMetadata derives its + // effective key from the legacy storage column plus the row text. A + // candidate query narrowed on metadata patterns alone never returns it, + // so the manual write reports "created" and both values stay active. + const dir = mkdtempSync(join(tmpdir(), "supersede-legacyrow-")); + const { context, store } = makeRealContext(dir); + try { + await store.store({ + text: "favorite drink: cola", + // Orthogonal to the embedder's constant [1, 0, 0]: the legacy row is + // invisible to the vector advisory, only the key scan can find it. + vector: [0, 1, 0], + category: "preference", + scope: "agent:main", + importance: 0.7, + metadata: "{}", + }); + + const storeTool = createToolSet(context).get("memory_store"); + const result = await storeTool.execute(null, { text: "favorite drink: tea", category: "preference" }); + assert.equal(result.details.action, "superseded"); + + const rows = await store.list(undefined, undefined, 100, 0); + const now = Date.now(); + const factKey = deriveFactKey("preferences", "favorite drink: cola"); + const activeSameKey = rows.filter((row) => { + const meta = parseSmartMetadata(row.metadata, row); + const key = meta.fact_key ?? deriveFactKey(meta.memory_category, row.text); + return key === factKey && isMemoryActiveAt(meta, now); + }); + assert.equal( + activeSameKey.length, + 1, + `exactly one active row may hold the derived key after the manual write (got: ${activeSameKey.map((row) => row.text).join(" | ")})`, + ); + assert.equal(activeSameKey[0].text, "favorite drink: tea", "the manual write must be the surviving active row"); + + const legacyRow = rows.find((row) => row.text === "favorite drink: cola"); + const legacyMeta = parseSmartMetadata(legacyRow.metadata, legacyRow); + assert.ok(legacyMeta.invalidated_at, "the legacy row must carry an invalidation timestamp"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("supersedes a whitespace-formatted explicit-key row stored under an unrelated storage category (end to end)", async () => { + // JSON.parse accepts arbitrary layout: '{"fact_key" : "..."}' resolves to + // the same normalized key but contains neither the compact '"fact_key":' + // substring nor a same-category stamp, and under storage category "fact" + // it dodges legacy-category widening too. Any narrowing built on raw + // serialization layout excludes it from collision discovery, so the + // manual write reports "created" and both query-equivalent values stay + // active. + const dir = mkdtempSync(join(tmpdir(), "supersede-spacedmeta-")); + const { context, store } = makeRealContext(dir); + try { + await store.store({ + text: "favorite drink: cola", + // Orthogonal to the embedder's constant [1, 0, 0]: invisible to the + // vector advisory, only the key scan can find it. + vector: [0, 1, 0], + category: "fact", + scope: "agent:main", + importance: 0.7, + metadata: + '{"fact_key" : "Preferences:Favorite Drink", "memory_category" : "preferences", "l0_abstract" : "favorite drink: cola"}', + }); + + const storeTool = createToolSet(context).get("memory_store"); + const result = await storeTool.execute(null, { text: "favorite drink: tea", category: "preference" }); + assert.equal(result.details.action, "superseded"); + + const rows = await store.list(undefined, undefined, 100, 0); + const now = Date.now(); + const activeSameKey = rows.filter((row) => { + const meta = parseSmartMetadata(row.metadata, row); + const key = (meta.fact_key ?? deriveFactKey(meta.memory_category, row.text) ?? "").trim().toLowerCase(); + return key === "preferences:favorite drink" && isMemoryActiveAt(meta, now); + }); + assert.equal( + activeSameKey.length, + 1, + `exactly one active row may hold the normalized key after the manual write (got: ${activeSameKey.map((row) => row.text).join(" | ")})`, + ); + assert.equal(activeSameKey[0].text, "favorite drink: tea", "the manual write must be the surviving active row"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe("fact-key collision scan cost and normalization", () => { + const fillerMetadata = JSON.stringify({ + memory_category: "preferences", + fact_key: "preferences:unrelated filler", + l0_abstract: "filler row", + source: "auto-capture", + state: "confirmed", + }); + + function fillerRows(count) { + const rows = []; + for (let i = 0; i < count; i += 1) { + rows.push({ + id: `filler-${i}`, + text: `filler row ${i}`, + category: "preference", + scope: "agent:main", + importance: 0.5, + timestamp: Date.now() - 120_000, + metadata: fillerMetadata, + }); + } + return rows; + } + + it("collision discovery issues one bounded candidate query per pass and never touches list()", async () => { + const collision = neighborRow({ + id: "old-key", + text: "favorite drink: Fizzwick", + score: 0.5, + factKey: "preferences:favorite drink", + }).entry; + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [], + rows: [...fillerRows(1200), collision], + }); + let listCalls = 0; + let scanCalls = 0; + const originalList = context.store.list.bind(context.store); + context.store.list = async (...args) => { + listCalls += 1; + return originalList(...args); + }; + const originalScan = context.store.listFactKeyCandidates.bind(context.store); + context.store.listFactKeyCandidates = async (...args) => { + scanCalls += 1; + return originalScan(...args); + }; + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: tea", category: "preference" }); + + assert.equal(res.details.action, "superseded", "the collision must still be found"); + assert.equal(storedEntries.length, 1); + assert.equal(patchCalls.length, 1); + assert.equal(patchCalls[0].id, "old-key"); + assert.equal( + scanCalls, + 2, + `discovery must run ONE bounded candidate query per pass (advisory + locked recheck); got ${scanCalls}`, + ); + assert.equal(listCalls, 0, "the collision scan must never fall back to the full-scope list()"); + }); + + it("supersedes a mixed-case explicit fact key that is query-equivalent to the derived key", async () => { + const legacyRow = neighborRow({ + id: "old-mixed-case", + text: "favorite drink: Fizzwick", + score: 0.5, + factKey: "Preferences:Favorite Drink", + }).entry; + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [], + rows: [legacyRow], + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const input = "favorite drink: tea"; + const res = await store.execute(null, { text: input, category: "preference" }); + + assert.equal( + res.details.action, + "superseded", + "a mixed-case explicit key must not evade supersession by its query-equivalent lowercase form", + ); + assert.equal(storedEntries.length, 1); + assert.equal(storedEntries[0].text, input); + assert.equal(patchCalls.length, 1); + assert.equal(patchCalls[0].id, "old-mixed-case"); + }); + + it("rejects the opted-in write explicitly when the candidate set exceeds the scan bound", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [], + rows: fillerRows(20_001), + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const res = await store.execute(null, { text: "favorite drink: tea", category: "preference" }); + + assert.equal( + res.details.action, + "rejected", + "an over-bound scan must reject the write, never silently store alongside an unfound old value", + ); + assert.equal(res.details.reason, "fact-key-scan-over-bound"); + assert.match(res.content[0].text, /force/i, "the rejection must name the force escape hatch"); + assert.equal(storedEntries.length, 0, "nothing may be written on a rejected supersede"); + assert.equal(patchCalls.length, 0); + }); + + it("force: true bypasses the collision scan and stores verbatim on an over-bound scope", async () => { + const { context, storedEntries, patchCalls } = makeContext({ + manualStoreSupersede: true, + neighbors: [], + rows: fillerRows(20_001), + }); + const tools = createToolSet(context); + const store = tools.get("memory_store"); + + const input = "favorite drink: tea"; + const res = await store.execute(null, { text: input, category: "preference", force: true }); + + assert.equal(res.details.action, "created", "force stores without entering the supersede path"); + assert.equal(storedEntries.length, 1, "force must always store"); + assert.equal(storedEntries[0].text, input, "the manual text must be stored verbatim"); + assert.equal(patchCalls.length, 0, "force never supersedes"); + }); + + it("real store: listFactKeyCandidates narrows database-side and honors its bound", async () => { + const dir = mkdtempSync(join(tmpdir(), "lancedb-factkey-scan-")); + try { + const realStore = new MemoryStore({ dbPath: dir, vectorDim: 3 }); + const seed = async (id, category, scope, metadata, text) => + realStore.store({ + text, + vector: [0.1, 0.2, 0.3], + importance: 0.5, + category, + scope, + metadata: JSON.stringify(metadata), + }); + await seed("a", "preference", "agent:main", { memory_category: "preferences", fact_key: "Preferences:Favorite Drink" }, "favorite drink: cola"); + await seed("b", "preference", "agent:main", { memory_category: "preferences" }, "favorite drink: tea"); + await seed("c", "fact", "agent:main", { memory_category: "events" }, "went to the market"); + await seed("d", "preference", "agent:other", { memory_category: "preferences", fact_key: "preferences:favorite drink" }, "favorite drink: soda"); + await seed("e", "preference", "agent:main", {}, "favorite drink: juice"); + await seed("f", "fact", "agent:main", {}, "favorite drink: fanta"); + + const candidates = await realStore.listFactKeyCandidates(["agent:main"], 10); + const texts = candidates.map((row) => row.text).sort(); + assert.deepEqual( + texts, + [ + "favorite drink: cola", + "favorite drink: fanta", + "favorite drink: juice", + "favorite drink: tea", + "went to the market", + ], + "candidates are every in-scope row regardless of metadata layout or category; other scopes stay out", + ); + + const bounded = await realStore.listFactKeyCandidates(["agent:main"], 1); + assert.equal(bounded.length, 2, "an over-bound candidate set returns exactly bound + 1 rows for detection"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +});