diff --git a/src/main/tape/application/common.ts b/src/main/tape/application/common.ts index 5f2fa5d24..898bab5ef 100644 --- a/src/main/tape/application/common.ts +++ b/src/main/tape/application/common.ts @@ -1,6 +1,5 @@ import { TAPE_INCARNATION_META_KEY, type DeepChatTapeEntryRow } from '../domain/entry' - -const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ +import { CANONICAL_UUID_PATTERN } from '../domain/primitives' export function parseJsonObject(raw: string): Record { try { @@ -24,7 +23,7 @@ export function readCanonicalTapeIncarnationId(row: DeepChatTapeEntryRow): strin return null } const value = parseJsonObject(row.meta_json)[TAPE_INCARNATION_META_KEY] - return typeof value === 'string' && UUID_PATTERN.test(value) ? value : null + return typeof value === 'string' && CANONICAL_UUID_PATTERN.test(value) ? value : null } export function parseJsonValue(raw: string): unknown { diff --git a/src/main/tape/application/factPersistence.ts b/src/main/tape/application/factPersistence.ts index d15a66e74..67d6642a2 100644 --- a/src/main/tape/application/factPersistence.ts +++ b/src/main/tape/application/factPersistence.ts @@ -7,7 +7,6 @@ import { } from '@/tape/domain/facts' import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' import type { TapeBootstrapStore, TapeEntryStore } from '@/tape/ports/storage' -import { buildEffectiveTapeView } from '@/tape/domain/effectiveView' import { parseAssistantBlocks, parseTapeJsonObject, @@ -612,9 +611,3 @@ export function appendMessageRetractionToTape( return 1 } - -export function tapeEntriesToEffectiveMessageRecords( - rows: DeepChatTapeEntryRow[] -): ChatMessageRecord[] { - return buildEffectiveTapeView(rows, { includePending: true }).messageRecords -} diff --git a/src/main/tape/application/factService.ts b/src/main/tape/application/factService.ts index 09081b3cf..0a961e703 100644 --- a/src/main/tape/application/factService.ts +++ b/src/main/tape/application/factService.ts @@ -281,7 +281,7 @@ export class TapeFactService } getMessageRecords(sessionId: string): ChatMessageRecord[] { - return buildEffectiveTapeView(this.table.getEffectiveViewInputRows(sessionId), { + return buildEffectiveTapeView(this.table.getEffectiveMessageInputRows(sessionId), { includePending: true }).messageRecords } diff --git a/src/main/tape/application/lineageService.ts b/src/main/tape/application/lineageService.ts index 567fa2801..9939737cc 100644 --- a/src/main/tape/application/lineageService.ts +++ b/src/main/tape/application/lineageService.ts @@ -11,7 +11,8 @@ import { } from '../domain/entry' import type { TapeApplicationProviders } from '../ports/application' import { parseJsonObject, parseJsonValue } from './common' -import { computeTapeIdentity, TAPE_IDENTITY_PATTERN } from '../domain/tapeIdentity' +import { SHA256_HEX_PATTERN } from '../domain/primitives' +import { computeTapeIdentity } from '../domain/tapeIdentity' type TapeLineageProviders = Pick< TapeApplicationProviders, @@ -141,7 +142,7 @@ function parseSubagentTapeLink(row: DeepChatTapeEntryRow): ParsedSubagentTapeLin (linkVersion === 1 && childTapeIdentity === undefined) || (linkVersion === SUBAGENT_TAPE_LINK_VERSION && typeof childTapeIdentity === 'string' && - TAPE_IDENTITY_PATTERN.test(childTapeIdentity)) + SHA256_HEX_PATTERN.test(childTapeIdentity)) if ( row.kind !== 'event' || row.name !== SUBAGENT_TAPE_LINK_EVENT_NAME || diff --git a/src/main/tape/application/viewReplayService.ts b/src/main/tape/application/viewReplayService.ts index 5abd28a7c..14d512d4a 100644 --- a/src/main/tape/application/viewReplayService.ts +++ b/src/main/tape/application/viewReplayService.ts @@ -15,7 +15,11 @@ import type { } from '@shared/types/tape-replay' import { SUMMARY_ANCHOR_NAMES, type DeepChatTapeEntryRow } from '../domain/entry' import { buildEffectiveTapeView } from '../domain/effectiveView' -import { readTapeMessageRetractionId, tapeEntryToMessageRecord } from '../domain/effectiveSemantics' +import { + isEffectiveMessageInputRow, + readTapeMessageRetractionId, + tapeEntryToMessageRecord +} from '../domain/effectiveSemantics' import { collectEntryIds, hashString, @@ -358,12 +362,10 @@ export class TapeViewReplayService { } } - const messageSourceRows = rows.filter( - (row) => row.kind === 'message' || (row.kind === 'event' && row.name === 'message/retracted') - ) - for (const { entryId, record } of buildEffectiveTapeView(messageSourceRows, { - includePending: true - }).messageEntries) { + for (const { entryId, record } of buildEffectiveTapeView( + rows.filter(isEffectiveMessageInputRow), + { includePending: true } + ).messageEntries) { entryIdByMessageId.set(record.id, entryId) if (record.role === 'user') { messageContentHashByMessageId.set(record.id, hashJsonData(record.content)) diff --git a/src/main/tape/domain/effectiveSemantics.ts b/src/main/tape/domain/effectiveSemantics.ts index 82a285c74..b4e9a5ab6 100644 --- a/src/main/tape/domain/effectiveSemantics.ts +++ b/src/main/tape/domain/effectiveSemantics.ts @@ -1,26 +1,58 @@ import type { AssistantMessageBlock, ChatMessageRecord } from '@shared/types/agent-interface' -import type { DeepChatTapeEntryRow } from './entry' +import type { DeepChatTapeEntryKind, DeepChatTapeEntryRow } from './entry' const TERMINAL_TAPE_TOOL_STATUSES = new Set(['success', 'error']) +export const TAPE_MESSAGE_RETRACTED_EVENT_NAME = 'message/retracted' + +/** + * Kinds an effective-state reader may select wholesale. `event` is excluded because the only event + * the effective view acts on is `message/retracted`, which every input set already selects by + * name (listing `event` would return those rows twice); `context` rows are behavioural evidence + * the fold skips outright. + */ +export type EffectiveInputKind = Exclude + +/** + * Rows that can change effective message/tool state or anchor positions, plus `message/retracted` + * events. Every other row (ViewManifests, Journal, provider attempts, contracts, tool-surface + * provenance, indicators) is evidence the effective view only passes through, so readers that need + * effective state skip it at the store. `TapeEntryStore.getEffectiveViewInputRows` selects by the + * same constant. + */ +export const EFFECTIVE_VIEW_INPUT_KINDS = [ + 'message', + 'tool_call', + 'tool_result', + 'anchor' +] as const satisfies readonly EffectiveInputKind[] + /** - * Rows that can change effective message/tool state or anchor positions. Every other row - * (ViewManifests, Journal, provider attempts, contracts, tool-surface provenance, indicators) is - * evidence the effective view only passes through, so readers that need effective state can skip - * it at the store. Must stay in sync with `TapeEntryStore.getEffectiveViewInputRows`. + * The subset that decides `messageRecords`/`messageEntries`: message rows plus the retraction + * events that remove them. Tool rows only join onto messages and anchors only pass through, so + * readers that need effective messages alone skip both at the store. + * `TapeEntryStore.getEffectiveMessageInputRows` selects by the same constant. */ +export const EFFECTIVE_MESSAGE_INPUT_KINDS = [ + 'message' +] as const satisfies readonly EffectiveInputKind[] + +function isEffectiveInputRow( + row: { kind: string; name: string | null }, + kinds: readonly string[] +): boolean { + return ( + kinds.includes(row.kind) || + (row.kind === 'event' && row.name === TAPE_MESSAGE_RETRACTED_EVENT_NAME) + ) +} + export function isEffectiveViewInputRow(row: { kind: string; name: string | null }): boolean { - switch (row.kind) { - case 'message': - case 'tool_call': - case 'tool_result': - case 'anchor': - return true - case 'event': - return row.name === 'message/retracted' - default: - return false - } + return isEffectiveInputRow(row, EFFECTIVE_VIEW_INPUT_KINDS) +} + +export function isEffectiveMessageInputRow(row: { kind: string; name: string | null }): boolean { + return isEffectiveInputRow(row, EFFECTIVE_MESSAGE_INPUT_KINDS) } export interface DeepChatTapeToolIdentity { @@ -130,7 +162,7 @@ export function tapeMessageRank(record: ChatMessageRecord, includePending: boole } export function readTapeMessageRetractionId(row: DeepChatTapeEntryRow): string | null { - if (row.kind !== 'event' || row.name !== 'message/retracted') { + if (row.kind !== 'event' || row.name !== TAPE_MESSAGE_RETRACTED_EVENT_NAME) { return null } @@ -144,38 +176,46 @@ export function readTapeToolStatus(row: DeepChatTapeEntryRow): string | null { return typeof meta.status === 'string' ? meta.status : null } -export function tapeToolRank(row: DeepChatTapeEntryRow, includePending: boolean): number { - const status = readTapeToolStatus(row) +export function tapeToolRankFromStatus(status: string | null, includePending: boolean): number { if (status === 'pending') { return includePending ? 1 : 0 } return status !== null && TERMINAL_TAPE_TOOL_STATUSES.has(status) ? 2 : 0 } -export function readTapeToolIdentity(row: DeepChatTapeEntryRow): DeepChatTapeToolIdentity | null { - if (row.kind !== 'tool_call' && row.kind !== 'tool_result') { +export function tapeToolRank(row: DeepChatTapeEntryRow, includePending: boolean): number { + return tapeToolRankFromStatus(readTapeToolStatus(row), includePending) +} + +/** `readTapeToolIdentity` for a caller that has already parsed `payload_json`. */ +export function readTapeToolIdentityFromPayload( + kind: DeepChatTapeEntryRow['kind'], + payload: Record +): DeepChatTapeToolIdentity | null { + if (kind !== 'tool_call' && kind !== 'tool_result') { return null } - const payload = parseTapeJsonObject(row.payload_json) const messageId = payload.messageId if (typeof messageId !== 'string' || messageId.length === 0) { return null } - let toolCallId: unknown - if (row.kind === 'tool_call') { - toolCallId = parseNestedTapeJsonObject(payload.toolCall).id - } else { - toolCallId = payload.toolCallId - } - + const toolCallId = + kind === 'tool_call' ? parseNestedTapeJsonObject(payload.toolCall).id : payload.toolCallId if (typeof toolCallId !== 'string' || toolCallId.length === 0) { return null } return { - key: `${row.kind}:${messageId}:${toolCallId}`, + key: `${kind}:${messageId}:${toolCallId}`, messageId } } + +export function readTapeToolIdentity(row: DeepChatTapeEntryRow): DeepChatTapeToolIdentity | null { + if (row.kind !== 'tool_call' && row.kind !== 'tool_result') { + return null + } + return readTapeToolIdentityFromPayload(row.kind, parseTapeJsonObject(row.payload_json)) +} diff --git a/src/main/tape/domain/effectiveView.ts b/src/main/tape/domain/effectiveView.ts index 536d3c80a..0667b0799 100644 --- a/src/main/tape/domain/effectiveView.ts +++ b/src/main/tape/domain/effectiveView.ts @@ -6,13 +6,15 @@ import { CONTRACT_TAPE_EVENT_NAMES, isContractTapeReservedName } from './contrac import { TOOL_SURFACE_TAPE_EVENT_NAMES } from './toolSurfaceFacts' import { TAPE_VIEW_MANIFEST_EVENT_NAME } from './viewManifest' import { + TAPE_MESSAGE_RETRACTED_EVENT_NAME, parseNestedTapeJsonObject, parseTapeJsonObject, readTapeMessageRetractionId, - readTapeToolIdentity, + readTapeToolIdentityFromPayload, + readTapeToolStatus, tapeEntryToMessageRecord, tapeMessageRank, - tapeToolRank + tapeToolRankFromStatus } from './effectiveSemantics' import { parseTapeProviderAttemptEvent, @@ -43,7 +45,7 @@ interface EffectiveTapeViewOptions { } export const DEFAULT_EXCLUDED_TAPE_EVENT_NAMES = [ - 'message/retracted', + TAPE_MESSAGE_RETRACTED_EVENT_NAME, 'message/compaction_indicator', 'migration/backfill', TAPE_COMPACTION_MODEL_CALL_EVENT_NAME, @@ -60,18 +62,25 @@ type EffectiveMessageCandidate = { record: ChatMessageRecord } +/** A tool row with everything the fold needs already parsed, so no JSON is parsed twice. */ +type EffectiveToolCandidate = { + row: DeepChatTapeEntryRow + messageId: string + rank: number + /** `payload.orderSeq` as stored; when it already matches the message, projection is a no-op. */ + storedOrderSeq: unknown +} + export function projectTapeToolOrderSeq( row: DeepChatTapeEntryRow, effectiveMessageOrderSeqById: ReadonlyMap ): DeepChatTapeEntryRow { - const identity = readTapeToolIdentity(row) + const payload = parseTapeJsonObject(row.payload_json) + const identity = readTapeToolIdentityFromPayload(row.kind, payload) if (!identity) return row const effectiveOrderSeq = effectiveMessageOrderSeqById.get(identity.messageId) - if (effectiveOrderSeq === undefined) return row - - const payload = parseTapeJsonObject(row.payload_json) - if (payload.orderSeq === effectiveOrderSeq) return row + if (effectiveOrderSeq === undefined || payload.orderSeq === effectiveOrderSeq) return row return { ...row, payload_json: JSON.stringify({ ...payload, orderSeq: effectiveOrderSeq }) @@ -132,23 +141,16 @@ function isAuditEvent(row: DeepChatTapeEntryRow): boolean { } function shouldReplaceToolRow( - current: DeepChatTapeEntryRow | undefined, - next: DeepChatTapeEntryRow, - includePending: boolean + current: EffectiveToolCandidate | undefined, + next: EffectiveToolCandidate ): boolean { if (!current) { return true } - - const currentRank = tapeToolRank(current, includePending) - const nextRank = tapeToolRank(next, includePending) - if (nextRank > currentRank) { - return true - } - if (nextRank < currentRank) { - return false + if (next.rank !== current.rank) { + return next.rank > current.rank } - return next.entry_id > current.entry_id + return next.row.entry_id > current.row.entry_id } function matchesKinds( @@ -184,7 +186,7 @@ export function buildEffectiveTapeView( const includeAuditEvents = options.includeAuditEvents === true const messageCandidates = new Map() const retractedMessageIds = new Set() - const toolRows = new Map() + const toolCandidates = new Map() const anchorRows: DeepChatTapeEntryRow[] = [] const eventRows: DeepChatTapeEntryRow[] = [] @@ -227,13 +229,18 @@ export function buildEffectiveTapeView( continue } - const identity = readTapeToolIdentity(row) - if (!identity || tapeToolRank(row, includePending) === 0) { + const payload = parseTapeJsonObject(row.payload_json) + const identity = readTapeToolIdentityFromPayload(row.kind, payload) + if (!identity) { continue } - const current = toolRows.get(identity.key)?.row - if (shouldReplaceToolRow(current, row, includePending)) { - toolRows.set(identity.key, { row, messageId: identity.messageId }) + const rank = tapeToolRankFromStatus(readTapeToolStatus(row), includePending) + if (rank === 0) { + continue + } + const candidate = { row, messageId: identity.messageId, rank, storedOrderSeq: payload.orderSeq } + if (shouldReplaceToolRow(toolCandidates.get(identity.key), candidate)) { + toolCandidates.set(identity.key, candidate) } } @@ -244,13 +251,21 @@ export function buildEffectiveTapeView( left.record.orderSeq - right.record.orderSeq || compareSqliteBinaryText(left.record.id, right.record.id) ) - const effectiveMessageIds = new Set(messageRows.map((candidate) => candidate.record.id)) const effectiveMessageOrderSeqById = new Map( messageRows.map((candidate) => [candidate.record.id, candidate.record.orderSeq]) ) - const effectiveToolRows = [...toolRows.values()] - .filter((candidate) => effectiveMessageIds.has(candidate.messageId)) - .map((candidate) => projectTapeToolOrderSeq(candidate.row, effectiveMessageOrderSeqById)) + // Tool rows only survive when their message did; the stored orderSeq matches unless a + // compaction shift moved the message, so the re-serialising projection is the rare path. + const effectiveToolRows: DeepChatTapeEntryRow[] = [] + for (const candidate of toolCandidates.values()) { + const effectiveOrderSeq = effectiveMessageOrderSeqById.get(candidate.messageId) + if (effectiveOrderSeq === undefined) continue + effectiveToolRows.push( + candidate.storedOrderSeq === effectiveOrderSeq + ? candidate.row + : projectTapeToolOrderSeq(candidate.row, effectiveMessageOrderSeqById) + ) + } const effectiveRows = [ ...anchorRows, ...eventRows, diff --git a/src/main/tape/domain/executionContract.ts b/src/main/tape/domain/executionContract.ts index d656da985..166f289a5 100644 --- a/src/main/tape/domain/executionContract.ts +++ b/src/main/tape/domain/executionContract.ts @@ -31,6 +31,13 @@ import { } from '@shared/types/execution-contract' import type { DeepChatTaskContractContext } from '@shared/types/task-contract' import { canonicalJsonStringifyData, hashJsonData } from './canonicalJson' +import { + canonicalUuid, + compareUtf16, + deepFreeze, + SHA256_HEX_PATTERN, + utf8Length +} from './primitives' import { isDeepChatTaskContract, isDeepChatTaskContractRef } from './taskContract' import { isWorkspacePathWithin, @@ -49,8 +56,6 @@ const MAX_SOURCE_REF_BYTES = 2_048 const MAX_WORKSPACE_PATH_BYTES = 32 * 1_024 const MAX_ASSEMBLER_VERSION_BYTES = 256 const MAX_SECTION_DEGRADATION_CODES = 16 -const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i -const SHA_256_PATTERN = /^[0-9a-f]{64}$/ const JSON_HASH_OPTIONS = Object.freeze({ omitUndefinedProperties: true }) const PROMPT_SECTION_KINDS = new Set(DEEPCHAT_PROMPT_SECTION_KINDS) const PROMPT_SECTION_INCLUSIONS = new Set(DEEPCHAT_PROMPT_SECTION_INCLUSIONS) @@ -218,10 +223,6 @@ export function restoreExecutionContract(value: unknown): DeepChatExecutionContr return isDeepChatExecutionContract(value) ? deepFreeze(value) : null } -function utf8Length(value: string): number { - return Buffer.byteLength(value, 'utf8') -} - function requireString( value: unknown, label: string, @@ -246,15 +247,15 @@ function requireString( } function requireUuid(value: unknown, label: string): string { - const uuid = requireString(value, label, MAX_IDENTITY_BYTES) - if (!UUID_PATTERN.test(uuid)) { + const uuid = canonicalUuid(requireString(value, label, MAX_IDENTITY_BYTES)) + if (!uuid) { throw new ExecutionContractError(`${label} must be a UUID.`, 'invalid_input') } - return uuid.toLowerCase() + return uuid } function requireSha256(value: unknown, label: string): string { - if (typeof value !== 'string' || !SHA_256_PATTERN.test(value)) { + if (typeof value !== 'string' || !SHA256_HEX_PATTERN.test(value)) { throw new ExecutionContractError(`${label} must be a lowercase SHA-256 hash.`, 'invalid_input') } return value @@ -335,11 +336,7 @@ function matchesNormalizedUuid(value: unknown, label: string): value is string { } function isSha256(value: unknown): value is string { - return typeof value === 'string' && SHA_256_PATTERN.test(value) -} - -function compareCodePoints(left: string, right: string): number { - return left < right ? -1 : left > right ? 1 : 0 + return typeof value === 'string' && SHA256_HEX_PATTERN.test(value) } function normalizeExecution(value: ToolExecutionContract, label: string): ToolExecutionContract { @@ -490,7 +487,7 @@ function normalizeToolCeilings( }) return [...ceilingByTarget.entries()] - .sort(([left], [right]) => compareCodePoints(left, right)) + .sort(([left], [right]) => compareUtf16(left, right)) .map(([, value]) => value.ceiling) } @@ -532,7 +529,7 @@ function normalizePromptSections( if (degradationCodes.some((code) => !PROMPT_DEGRADATION_CODES.has(code))) { throw new ExecutionContractError(`${label}.degradationCodes is invalid.`, 'invalid_input') } - degradationCodes.sort(compareCodePoints) + degradationCodes.sort(compareUtf16) if (section.freshness !== undefined && !PROMPT_SOURCE_FRESHNESS_VALUES.has(section.freshness)) { throw new ExecutionContractError(`${label}.freshness is invalid.`, 'invalid_input') } @@ -726,7 +723,7 @@ function isStoredPromptSection(value: unknown): value is DeepChatPromptSectionPr (code, index) => typeof code !== 'string' || !PROMPT_DEGRADATION_CODES.has(code) || - (index > 0 && compareCodePoints(degradationCodes[index - 1], code) >= 0) + (index > 0 && compareUtf16(degradationCodes[index - 1], code) >= 0) ) ) { return false @@ -779,7 +776,7 @@ function isStoredExecutionCeilings(value: unknown): value is DeepChatExecutionCo return false } const targetKey = buildExecutionToolTargetKey(tool.target) - if (previousTargetKey !== null && compareCodePoints(previousTargetKey, targetKey) >= 0) { + if (previousTargetKey !== null && compareUtf16(previousTargetKey, targetKey) >= 0) { return false } const previousVisibleTarget = targetKeyByVisibleName.get(tool.target.providerVisibleName) @@ -1041,14 +1038,6 @@ export function assertExecutionContractAllowsDispatch( } } -function deepFreeze(value: T): T { - if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value - for (const nested of Object.values(value as Record)) { - deepFreeze(nested) - } - return Object.freeze(value) -} - function buildContractHash(contract: Omit): string { return hashData(contract, 'execution contract') } @@ -1112,7 +1101,7 @@ export function verifyExecutionContractHash(contract: DeepChatExecutionContract) contract?.schemaVersion !== DEEPCHAT_EXECUTION_CONTRACT_SCHEMA_VERSION || contract?.hashVersion !== DEEPCHAT_EXECUTION_CONTRACT_HASH_VERSION || typeof contract.contractHash !== 'string' || - !SHA_256_PATTERN.test(contract.contractHash) + !SHA256_HEX_PATTERN.test(contract.contractHash) ) { return false } diff --git a/src/main/tape/domain/executionJournal.ts b/src/main/tape/domain/executionJournal.ts index 68dfbf81b..ae48279ac 100644 --- a/src/main/tape/domain/executionJournal.ts +++ b/src/main/tape/domain/executionJournal.ts @@ -1,5 +1,6 @@ import type { DeepChatTapeEntryRow } from './entry' import { hashJson, hashJsonData, stableJsonStringify } from './canonicalJson' +import { canonicalUuid, SHA256_HEX_PATTERN } from './primitives' export const EXECUTION_JOURNAL_PROTOCOL_VERSION = 1 as const export const EXECUTION_JOURNAL_NESTED_PROTOCOL_VERSION = 2 as const export const MAX_EXECUTION_JOURNAL_NESTED_CHILDREN = 128 @@ -33,8 +34,6 @@ const MAX_IDENTITY_CHARS = 1_024 export const MAX_EXECUTION_JOURNAL_TOOL_NAME_CHARACTERS = 512 const MAX_TARGET_FIELD_CHARS = 1_024 const MAX_STOP_REASON_CHARS = 1_024 -const SHA_256_PATTERN = /^[0-9a-f]{64}$/ -const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i export interface ExecutionOperationIdentity { runId: string @@ -287,11 +286,11 @@ function requireMessageId(value: unknown): string { } export function requireExecutionRunId(value: unknown): string { - const runId = requireString(value, 'runId', MAX_IDENTITY_CHARS) - if (!UUID_PATTERN.test(runId)) { + const runId = canonicalUuid(requireString(value, 'runId', MAX_IDENTITY_CHARS)) + if (!runId) { throw new ExecutionJournalError('runId must be a UUID.', 'invalid_fact') } - return runId.toLowerCase() + return runId } function requireRequestSeq(value: unknown): number { @@ -435,7 +434,7 @@ function requireProtocolVersion(value: unknown): ExecutionJournalProtocolVersion } function requireHash(value: unknown, label: string): string { - if (typeof value !== 'string' || !SHA_256_PATTERN.test(value)) { + if (typeof value !== 'string' || !SHA256_HEX_PATTERN.test(value)) { throw new ExecutionJournalError(`${label} must be a lowercase SHA-256 hash.`, 'invalid_fact') } return value diff --git a/src/main/tape/domain/primitives.ts b/src/main/tape/domain/primitives.ts new file mode 100644 index 000000000..f6758e8e7 --- /dev/null +++ b/src/main/tape/domain/primitives.ts @@ -0,0 +1,36 @@ +/** Lower-case SHA-256 hex digest; the shape every stored Tape hash is validated against. */ +export const SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/ + +/** + * RFC 9562 UUID (versions 1-8) in canonical lower-case form; the shape every UUID identity is + * stored in. Validate stored values against it directly; normalise input with `canonicalUuid`. + */ +export const CANONICAL_UUID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ + +/** The canonical form of a UUID given in any letter case, or null when it is not a UUID. */ +export function canonicalUuid(value: string): string | null { + const lower = value.toLowerCase() + return CANONICAL_UUID_PATTERN.test(lower) ? lower : null +} + +/** Tape size limits are byte limits on the UTF-8 encoding, not JavaScript string lengths. */ +export function utf8Length(value: string): number { + return Buffer.byteLength(value, 'utf8') +} + +/** + * Orders strings by UTF-16 code unit, which is what `<` does. Canonical key and target orderings + * inside persisted facts are defined on this comparison; a locale or code-point compare would + * reorder existing hashes. + */ +export function compareUtf16(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0 +} + +/** Freezes a plain-data tree in place and returns it; already-frozen subtrees are not revisited. */ +export function deepFreeze(value: T): T { + if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value + for (const nested of Object.values(value as Record)) deepFreeze(nested) + return Object.freeze(value) +} diff --git a/src/main/tape/domain/replay.ts b/src/main/tape/domain/replay.ts index 27952e458..6bd1e2adb 100644 --- a/src/main/tape/domain/replay.ts +++ b/src/main/tape/domain/replay.ts @@ -8,6 +8,7 @@ import { isDeepChatExecutionContract } from './executionContract' import { hashJson } from './viewManifest' import { validateSchema6SkillContexts, validateSchema7SkillContexts } from './skillContext' import { isBoundedSkillTapeIdentity } from './skillIdentity' +import { SHA256_HEX_PATTERN } from './primitives' const VIEW_POLICIES = new Set([ 'cache_aware_context_v2', @@ -36,7 +37,6 @@ const SCHEMA_V3_ENTRY_REASONS = new Set([ 'reconstruction_checkpoint', 'memory_context' ]) -const SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/ const VIEW_EXCLUDED_REASONS = new Set([ 'before_summary_cursor', diff --git a/src/main/tape/domain/skillContext.ts b/src/main/tape/domain/skillContext.ts index 3823818f3..e3dafc536 100644 --- a/src/main/tape/domain/skillContext.ts +++ b/src/main/tape/domain/skillContext.ts @@ -16,8 +16,8 @@ import { } from './executionJournal' import { hashJsonData } from './canonicalJson' import { isBoundedSkillTapeIdentity } from './skillIdentity' +import { SHA256_HEX_PATTERN } from './primitives' -const HASH = /^[a-f0-9]{64}$/ const MAX_SOURCE_REFS = 64 export const MAX_SKILL_CONTEXTS_PER_VIEW = 64 export const MAX_SKILL_VIEW_RESULT_FACT_BYTES = SKILL_RUNTIME_VIEW_RESULT_MAX_BYTES @@ -219,7 +219,7 @@ function validateMaterializationRef( !isSkillSourceType(ref.sourceType) || !isBoundedSkillTapeIdentity(ref.sourceId) || !isBoundedSkillTapeIdentity(ref.skillName) || - !HASH.test(ref.effectiveContentHash) || + !SHA256_HEX_PATTERN.test(ref.effectiveContentHash) || (projectedContentHash !== undefined && projectedContentHash !== ref.effectiveContentHash) || ref.agentId !== context.agentId || ref.sourceType !== context.sourceType || @@ -270,7 +270,7 @@ function validateSkillContexts( !ref || !positive(ref.entryId) || typeof context.projectedContentHash !== 'string' || - !HASH.test(context.projectedContentHash) || + !SHA256_HEX_PATTERN.test(context.projectedContentHash) || !positive(context.projectionVersion) || !Array.isArray(context.sourceEntryIds) || context.sourceEntryIds.length > MAX_SOURCE_REFS || @@ -296,7 +296,7 @@ function validateSkillContexts( context.providerRole !== 'tool' || ref.kind !== 'tool_result' || typeof ref.contentHash !== 'string' || - !HASH.test(ref.contentHash) || + !SHA256_HEX_PATTERN.test(ref.contentHash) || context.projectedContentHash !== ref.contentHash || context.deduplicationSource !== 'runtime_view' ) diff --git a/src/main/tape/domain/skillMaterialization.ts b/src/main/tape/domain/skillMaterialization.ts index 21ca83049..9d5028873 100644 --- a/src/main/tape/domain/skillMaterialization.ts +++ b/src/main/tape/domain/skillMaterialization.ts @@ -18,6 +18,7 @@ import { } from '@shared/types/skill' import type { DeepChatTapeEntryRow } from './entry' import { isBoundedSkillTapeIdentity, MAX_SKILL_TAPE_IDENTITY_BYTES } from './skillIdentity' +import { SHA256_HEX_PATTERN } from './primitives' export const SKILL_MATERIALIZATION_NAME = 'skill/materialized' as const export const SKILL_MATERIALIZATION_SCHEMA_VERSION = 3 as const @@ -32,8 +33,6 @@ export const MAX_SKILL_MATERIALIZATION_PACKAGE_BATCH_ENCODED_BYTES = const MAX_SKILL_MATERIALIZATION_STORED_PAYLOAD_BYTES = SKILL_EFFECTIVE_CONTENT_MAX_BYTES * 6 + SKILL_EXECUTION_PACKAGE_MAX_ENCODED_BYTES + 64 * 1024 -const SHA256 = /^[a-f0-9]{64}$/ - export interface TapeSkillIdentity { agentId: string sourceType: SkillSourceType @@ -99,7 +98,7 @@ function requireIdentity(value: unknown, field: string): string { } function requireHash(value: unknown, field: string): string { - if (typeof value !== 'string' || !SHA256.test(value)) { + if (typeof value !== 'string' || !SHA256_HEX_PATTERN.test(value)) { throw new TypeError(`${field} must be a lowercase SHA-256 hash.`) } return value diff --git a/src/main/tape/domain/tapeIdentity.ts b/src/main/tape/domain/tapeIdentity.ts index 56b71a527..dfab914ef 100644 --- a/src/main/tape/domain/tapeIdentity.ts +++ b/src/main/tape/domain/tapeIdentity.ts @@ -1,8 +1,6 @@ import { createHash } from 'node:crypto' import type { DeepChatTapeEntryRow } from './entry' -export const TAPE_IDENTITY_PATTERN = /^[a-f0-9]{64}$/u - export function computeTapeIdentity(row: DeepChatTapeEntryRow): string { return createHash('sha256') .update( diff --git a/src/main/tape/domain/taskContract.ts b/src/main/tape/domain/taskContract.ts index 46764e395..43f98afbd 100644 --- a/src/main/tape/domain/taskContract.ts +++ b/src/main/tape/domain/taskContract.ts @@ -1,4 +1,3 @@ -import { Buffer } from 'node:buffer' import { DEEPCHAT_TASK_CONTRACT_HASH_VERSION, DEEPCHAT_TASK_CONTRACT_SCHEMA_VERSION, @@ -11,6 +10,7 @@ import { type DeepChatTaskWorkspaceCeiling } from '@shared/types/task-contract' import { canonicalJsonStringifyData, hashJsonData } from './canonicalJson' +import { compareUtf16, deepFreeze, SHA256_HEX_PATTERN, utf8Length } from './primitives' import { normalizeAbsoluteWorkspacePath } from './workspacePath' const MAX_IDENTITY_BYTES = 1_024 @@ -20,7 +20,6 @@ const MAX_SECTION_NAME_BYTES = 256 const MAX_WORKSPACE_PATH_BYTES = 32 * 1024 const MAX_TASK_INPUT_BYTES = 64 * 1024 const MAX_SUBAGENT_DEPTH = 1 -const SHA_256_PATTERN = /^[0-9a-f]{64}$/u const TASK_CONTRACT_KEYS = [ 'schemaVersion', @@ -69,10 +68,6 @@ export class TaskContractError extends Error { } } -function utf8Length(value: string): number { - return Buffer.byteLength(value, 'utf8') -} - function requireString( value: unknown, label: string, @@ -111,10 +106,6 @@ function requireNonNegativeSafeInteger(value: unknown, label: string): number { return value as number } -function compareCodePoints(left: string, right: string): number { - return left < right ? -1 : left > right ? 1 : 0 -} - function normalizeWorkspace(workspace: DeepChatTaskWorkspaceCeiling): DeepChatTaskWorkspaceCeiling { if (workspace?.kind === 'runtime_default') return { kind: 'runtime_default' } if (workspace?.kind !== 'path' || typeof workspace.path !== 'string') { @@ -143,8 +134,8 @@ function normalizeEvaluationRef(value: DeepChatEvaluationRef | null): DeepChatEv const entryId = requirePositiveSafeInteger(value?.entryId, 'predecessorEvaluationRef.entryId') if ( value?.schemaVersion !== 1 || - !SHA_256_PATTERN.test(value.tapeIdentity) || - !SHA_256_PATTERN.test(value.evaluationHash) + !SHA256_HEX_PATTERN.test(value.tapeIdentity) || + !SHA256_HEX_PATTERN.test(value.evaluationHash) ) { throw new TaskContractError('predecessorEvaluationRef is invalid.', 'invalid_input') } @@ -207,16 +198,10 @@ function normalizeHandoffFormat( if (sections.length === 0 || sections.length > MAX_TASK_CONTRACT_REQUIREMENTS) { throw new TaskContractError(`${label}.sections has an invalid size.`, 'invalid_input') } - sections.sort(compareCodePoints) + sections.sort(compareUtf16) return { id, kind: 'required_sections' as const, level: 2 as const, sections } }) - return normalized.sort((left, right) => compareCodePoints(left.id, right.id)) -} - -function deepFreeze(value: T): T { - if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value - for (const nested of Object.values(value as Record)) deepFreeze(nested) - return Object.freeze(value) + return normalized.sort((left, right) => compareUtf16(left.id, right.id)) } function hasExactKeys(value: unknown, keys: readonly string[]): value is Record { @@ -319,7 +304,7 @@ export function isDeepChatTaskContract(value: unknown): value is DeepChatTaskCon value.schemaVersion !== DEEPCHAT_TASK_CONTRACT_SCHEMA_VERSION || value.hashVersion !== DEEPCHAT_TASK_CONTRACT_HASH_VERSION || typeof value.contractHash !== 'string' || - !SHA_256_PATTERN.test(value.contractHash) + !SHA256_HEX_PATTERN.test(value.contractHash) ) { return false } @@ -365,9 +350,9 @@ export function isDeepChatTaskContractRef(value: unknown): value is DeepChatTask requireString(value.sessionId, 'TaskContractRef.sessionId', MAX_IDENTITY_BYTES, 256) === value.sessionId && typeof value.tapeIdentity === 'string' && - SHA_256_PATTERN.test(value.tapeIdentity) && + SHA256_HEX_PATTERN.test(value.tapeIdentity) && typeof value.contractHash === 'string' && - SHA_256_PATTERN.test(value.contractHash) && + SHA256_HEX_PATTERN.test(value.contractHash) && requirePositiveSafeInteger(value.entryId, 'TaskContractRef.entryId') === value.entryId ) } catch { diff --git a/src/main/tape/domain/taskEvaluation.ts b/src/main/tape/domain/taskEvaluation.ts index 4bba818ef..6e26c805f 100644 --- a/src/main/tape/domain/taskEvaluation.ts +++ b/src/main/tape/domain/taskEvaluation.ts @@ -26,9 +26,9 @@ import { } from '@shared/types/task-contract' import { indexMarkdownLevelTwoSections } from '@shared/orchestration/liveDelegationMarkdown' import { canonicalJsonStringifyData, hashJsonData } from './canonicalJson' +import { compareUtf16, deepFreeze, SHA256_HEX_PATTERN } from './primitives' import { isDeepChatTaskContract } from './taskContract' -const SHA_256_PATTERN = /^[0-9a-f]{64}$/u const SUCCESS_REASON_CODES = new Set([ 'required_sections_present' ]) @@ -105,7 +105,7 @@ export function buildTaskEvaluation(input: BuildTaskEvaluationInput): DeepChatTa : 'valid' const reasonCodes = [ ...new Set(records.filter((record) => record.outcome !== 'valid').map((record) => record.code)) - ].sort(compareCodePoints) + ].sort(compareUtf16) return finalizeEvaluation({ schemaVersion: DEEPCHAT_TASK_EVALUATION_SCHEMA_VERSION, @@ -179,11 +179,11 @@ export function isDeepChatEvaluationRef(value: unknown): value is DeepChatEvalua ref.sessionId.length > 0 && ref.sessionId.length <= 256 && typeof ref.tapeIdentity === 'string' && - SHA_256_PATTERN.test(ref.tapeIdentity) && + SHA256_HEX_PATTERN.test(ref.tapeIdentity) && Number.isSafeInteger(ref.entryId) && (ref.entryId as number) > 0 && typeof ref.evaluationHash === 'string' && - SHA_256_PATTERN.test(ref.evaluationHash) + SHA256_HEX_PATTERN.test(ref.evaluationHash) ) } @@ -345,7 +345,7 @@ function isCanonicalEvaluation(evaluation: DeepChatTaskEvaluation): boolean { if (evaluation.reasonCodes.some((code) => SUCCESS_REASON_CODES.has(code))) return false if ( canonicalJsonStringifyData(evaluation.reasonCodes) !== - canonicalJsonStringifyData([...new Set(evaluation.reasonCodes)].sort(compareCodePoints)) + canonicalJsonStringifyData([...new Set(evaluation.reasonCodes)].sort(compareUtf16)) ) { return false } @@ -353,7 +353,7 @@ function isCanonicalEvaluation(evaluation: DeepChatTaskEvaluation): boolean { ...new Set( evaluation.records.filter((record) => record.outcome !== 'valid').map((record) => record.code) ) - ].sort(compareCodePoints) + ].sort(compareUtf16) if ( evaluation.omittedRecordCount === 0 && canonicalJsonStringifyData(evaluation.reasonCodes) !== @@ -380,7 +380,7 @@ function isCanonicalLegacyEvaluation(evaluation: DeepChatLegacyTaskEvaluation): if (evaluation.reasonCodes.some((code) => LEGACY_SUCCESS_REASON_CODES.has(code))) return false if ( canonicalJsonStringifyData(evaluation.reasonCodes) !== - canonicalJsonStringifyData([...new Set(evaluation.reasonCodes)].sort(compareCodePoints)) + canonicalJsonStringifyData([...new Set(evaluation.reasonCodes)].sort(compareUtf16)) ) { return false } @@ -390,7 +390,7 @@ function isCanonicalLegacyEvaluation(evaluation: DeepChatLegacyTaskEvaluation): .filter((record) => record.outcome !== 'passed') .map((record) => record.code) ) - ].sort(compareCodePoints) + ].sort(compareUtf16) if ( evaluation.omittedRecordCount === 0 && canonicalJsonStringifyData(evaluation.reasonCodes) !== @@ -450,7 +450,7 @@ function legacyEvaluationMatchesWriterState(evaluation: DeepChatLegacyTaskEvalua record.requirementKind !== 'required_sections' || requirementIds.has(record.requirementId) || (previousRequirementId !== null && - compareCodePoints(previousRequirementId, record.requirementId) >= 0) + compareUtf16(previousRequirementId, record.requirementId) >= 0) ) { return false } @@ -532,13 +532,3 @@ function reasonCodeOutcome( if (code === 'required_sections_missing') return 'invalid' return 'indeterminate' } - -function compareCodePoints(left: string, right: string): number { - return left < right ? -1 : left > right ? 1 : 0 -} - -function deepFreeze(value: T): T { - if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value - for (const nested of Object.values(value as Record)) deepFreeze(nested) - return Object.freeze(value) -} diff --git a/src/main/tape/domain/toolSurfaceFacts.ts b/src/main/tape/domain/toolSurfaceFacts.ts index ed68fa84d..c92e1af64 100644 --- a/src/main/tape/domain/toolSurfaceFacts.ts +++ b/src/main/tape/domain/toolSurfaceFacts.ts @@ -12,6 +12,7 @@ import type { DeepChatExecutionToolTargetIdentity } from '@shared/types/executio import type { DeepChatTaskContractRef } from '@shared/types/task-contract' import { canonicalJsonStringifyData, hashJsonData } from './canonicalJson' import { buildExecutionToolTargetKey, isDetachedStoredToolTarget } from './executionContract' +import { CANONICAL_UUID_PATTERN, compareUtf16, deepFreeze, SHA256_HEX_PATTERN } from './primitives' import { isDeepChatTaskContractRef } from './taskContract' import { normalizeAbsoluteWorkspacePath } from './workspacePath' @@ -62,8 +63,6 @@ const MAX_VERSION_BYTES = 256 const MAX_PROGRAMMATIC_POLICY_VERSION_BYTES = MAX_IDENTITY_BYTES const MAX_PLAIN_DATA_DEPTH = 64 const MAX_PLAIN_DATA_NODES = 100_000 -const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/ -const SHA_256_PATTERN = /^[a-f0-9]{64}$/ const MODEL_EXPOSURES = new Set(['user-configurable', 'system-model']) const TOOL_SURFACE_ADAPTER_MODES = new Set([ 'direct-native', @@ -454,11 +453,11 @@ function isNormalizedBoundedString(value: unknown, maxBytes = MAX_IDENTITY_BYTES } function isHash(value: unknown): value is string { - return typeof value === 'string' && SHA_256_PATTERN.test(value) + return typeof value === 'string' && SHA256_HEX_PATTERN.test(value) } function isUuid(value: unknown): value is string { - return typeof value === 'string' && UUID_PATTERN.test(value) + return typeof value === 'string' && CANONICAL_UUID_PATTERN.test(value) } function isToolSurfaceAdapterMode(value: unknown): value is TapeToolSurfaceAdapterMode { @@ -536,10 +535,6 @@ function cloneCatalogEntry(entry: TapeToolCatalogSourceEntry): TapeToolCatalogSo return cloneCatalogEntryFields(entry) } -function compareCodePoints(left: string, right: string): number { - return left < right ? -1 : left > right ? 1 : 0 -} - function catalogProjectionHash(input: { fullCatalogHash: string totalEntryCount: number @@ -633,12 +628,6 @@ function findLargestFittingPrefix(maximum: number, build: (length: number) => un return lower } -function deepFreeze(value: T): T { - if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value - for (const nested of Object.values(value as Record)) deepFreeze(nested) - return Object.freeze(value) -} - export function createTapeToolCatalogFact( input: CreateTapeToolCatalogFactInput ): TapeToolCatalogFact { @@ -662,7 +651,7 @@ function createTapeToolCatalogFactFromData( const entries = input.entries .map(cloneCatalogEntry) - .sort((left, right) => compareCodePoints(left.stableTargetKey, right.stableTargetKey)) + .sort((left, right) => compareUtf16(left.stableTargetKey, right.stableTargetKey)) const targetByVisibleName = new Map() for (let index = 0; index < entries.length; index += 1) { if (index > 0 && entries[index - 1].stableTargetKey === entries[index].stableTargetKey) { @@ -749,7 +738,7 @@ function isCatalogFactShape(value: unknown): value is TapeToolCatalogFact { for (let index = 0; index < value.entries.length; index += 1) { if ( index > 0 && - compareCodePoints( + compareUtf16( value.entries[index - 1].stableTargetKey, value.entries[index].stableTargetKey ) >= 0 @@ -1102,7 +1091,7 @@ function compareSearchRefs( left.originRequestSeq - right.originRequestSeq || left.toolCallOrdinalWithinBatch - right.toolCallOrdinalWithinBatch || left.resultRank - right.resultRank || - compareCodePoints(left.stableTargetKey, right.stableTargetKey) + compareUtf16(left.stableTargetKey, right.stableTargetKey) ) } @@ -1201,8 +1190,8 @@ function requireCompleteAcceptedSearchProvenance( const expectedTargets = activeEntries .filter((entry) => entry.reason === 'search-result') .map((entry) => entry.stableTargetKey) - .sort(compareCodePoints) - const actualTargets = searchResultRefs.map((ref) => ref.stableTargetKey).sort(compareCodePoints) + .sort(compareUtf16) + const actualTargets = searchResultRefs.map((ref) => ref.stableTargetKey).sort(compareUtf16) if (canonicalJsonStringifyData(expectedTargets) !== canonicalJsonStringifyData(actualTargets)) { throw new TypeError('Tool surface is missing accepted ToolSearch result provenance.') } @@ -1717,7 +1706,7 @@ function validateProgrammaticEntries(entries: readonly TapeToolCatalogSourceEntr ) } if (index > 0) { - const order = compareCodePoints(entries[index - 1].stableTargetKey, entry.stableTargetKey) + const order = compareUtf16(entries[index - 1].stableTargetKey, entry.stableTargetKey) if (order === 0) { throw new TypeError('Programmatic Tool Surface contains a duplicate stable target.') } @@ -1785,7 +1774,7 @@ export function createTapeProgrammaticToolSurfaceFact( } const entries = data.entries .map(cloneCatalogEntry) - .sort((left, right) => compareCodePoints(left.stableTargetKey, right.stableTargetKey)) + .sort((left, right) => compareUtf16(left.stableTargetKey, right.stableTargetKey)) validateProgrammaticEntries(entries) if ( buildProgrammaticToolSurfaceHashV1({ diff --git a/src/main/tape/domain/viewManifest.ts b/src/main/tape/domain/viewManifest.ts index 07996e106..ff0057e4a 100644 --- a/src/main/tape/domain/viewManifest.ts +++ b/src/main/tape/domain/viewManifest.ts @@ -23,6 +23,7 @@ import type { import { validateSchema6SkillContexts, validateSchema7SkillContexts } from './skillContext' import { estimateMessagesTokens } from '@shared/utils/messageTokens' import { hashJson, hashJsonData } from './canonicalJson' +import { SHA256_HEX_PATTERN } from './primitives' import { buildProviderMessagesHash, buildProviderVisibleToolDefinitionsHash, @@ -50,7 +51,6 @@ export function isCompactionRecord(record: ChatMessageRecord): boolean { /** Stable event name persisted for deterministic view reconstruction. */ export const TAPE_VIEW_MANIFEST_EVENT_NAME = 'view/assembled' export const TAPE_VIEW_CONTEXT_BUILDER_VERSION = 'cache-aware-v2' as const -const SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/ export function getTapeViewManifestExecutionContract( manifest: DeepChatTapeViewManifest diff --git a/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts b/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts index 034557156..e1a067d11 100644 --- a/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts +++ b/src/main/tape/infrastructure/sqlite/tapeEntryStore.ts @@ -16,7 +16,13 @@ import { type TapeEventAppendInput } from '@/tape/domain/entry' import { DEFAULT_EXCLUDED_TAPE_EVENT_NAMES } from '@/tape/domain/effectiveView' -import { parseTapeJsonObject } from '@/tape/domain/effectiveSemantics' +import { + EFFECTIVE_MESSAGE_INPUT_KINDS, + EFFECTIVE_VIEW_INPUT_KINDS, + TAPE_MESSAGE_RETRACTED_EVENT_NAME, + parseTapeJsonObject, + type EffectiveInputKind +} from '@/tape/domain/effectiveSemantics' import { EXECUTION_JOURNAL_EVENT_NAMES, type ExecutionJournalEventName, @@ -249,6 +255,32 @@ const EXECUTION_JOURNAL_EVENT_NAMES_SQL = EXECUTION_JOURNAL_EVENT_NAMES.map( (name) => `'${name}'` ).join(', ') +/** + * One session's rows of the given kinds plus its `message/retracted` events (every effective-state + * reader needs the retractions), merged in entry_id order; bind `{ session }`. Written as + * `WHERE kind IN (...)` the planner walks the whole session by primary key and, under SQLCipher, + * decrypts every page the session touches. One range per kind on + * `idx_deepchat_tape_entries_session_kind` (retractions on the partial event-name index) loads only + * the pages that hold the requested rows, and SQLite merges the ordered ranges without a sort. On a + * 10k-entry encrypted session this halves the cold read of the effective-view inputs. The ranges + * are disjoint because `EffectiveInputKind` cannot name `event`; the kind lists are the same + * constants the JS predicates test. + */ +function effectiveInputRowsSql(kinds: readonly EffectiveInputKind[]): string { + const ranges = [ + ...kinds.map( + (kind) => + `SELECT * FROM deepchat_tape_entries WHERE session_id = $session AND kind = '${kind}'` + ), + `SELECT * FROM deepchat_tape_entries + WHERE session_id = $session AND kind = 'event' AND name = '${TAPE_MESSAGE_RETRACTED_EVENT_NAME}'` + ] + return `SELECT * FROM (${ranges.join(' UNION ALL ')}) ORDER BY entry_id ASC` +} + +const EFFECTIVE_VIEW_INPUT_ROWS_SQL = effectiveInputRowsSql(EFFECTIVE_VIEW_INPUT_KINDS) +const EFFECTIVE_MESSAGE_INPUT_ROWS_SQL = effectiveInputRowsSql(EFFECTIVE_MESSAGE_INPUT_KINDS) + export const UNTERMINATED_EXECUTION_JOURNAL_EVENTS_SQL = ` WITH unterminated_runs AS ( SELECT DISTINCT started.session_id, started.source_id AS run_id @@ -1201,20 +1233,16 @@ export class DeepChatTapeEntriesTable .all(sessionId) as DeepChatTapeEntryRow[] } - /** SQL mirror of `isEffectiveViewInputRow`; skips the bulky rows the effective view ignores. */ getEffectiveViewInputRows(sessionId: string): DeepChatTapeEntryRow[] { return this.db - .prepare( - `SELECT * - FROM deepchat_tape_entries - WHERE session_id = ? - AND ( - kind IN ('message', 'tool_call', 'tool_result', 'anchor') - OR (kind = 'event' AND name = 'message/retracted') - ) - ORDER BY entry_id ASC` - ) - .all(sessionId) as DeepChatTapeEntryRow[] + .prepare(EFFECTIVE_VIEW_INPUT_ROWS_SQL) + .all({ session: sessionId }) as DeepChatTapeEntryRow[] + } + + getEffectiveMessageInputRows(sessionId: string): DeepChatTapeEntryRow[] { + return this.db + .prepare(EFFECTIVE_MESSAGE_INPUT_ROWS_SQL) + .all({ session: sessionId }) as DeepChatTapeEntryRow[] } getByEntryIds(sessionId: string, entryIds: readonly number[]): DeepChatTapeEntryRow[] { diff --git a/src/main/tape/ports/storage.ts b/src/main/tape/ports/storage.ts index beb446916..4b837f89b 100644 --- a/src/main/tape/ports/storage.ts +++ b/src/main/tape/ports/storage.ts @@ -89,6 +89,8 @@ export interface TapeEntryStore { getBySessionExcludingContext(sessionId: string): DeepChatTapeEntryRow[] /** Rows selected by `isEffectiveViewInputRow`, ordered by entry_id. */ getEffectiveViewInputRows(sessionId: string): DeepChatTapeEntryRow[] + /** Rows selected by `isEffectiveMessageInputRow`, ordered by entry_id. */ + getEffectiveMessageInputRows(sessionId: string): DeepChatTapeEntryRow[] getByEntryIds(sessionId: string, entryIds: readonly number[]): DeepChatTapeEntryRow[] getMessageSourceEntries(sessionId: string, messageId: string): DeepChatTapeEntryRow[] getLatestViewManifestEvent(sessionId: string): DeepChatTapeEntryRow | undefined diff --git a/test/main/agent/acp/compatibility/adapters.test.ts b/test/main/agent/acp/compatibility/adapters.test.ts index aeee81511..542751f0d 100644 --- a/test/main/agent/acp/compatibility/adapters.test.ts +++ b/test/main/agent/acp/compatibility/adapters.test.ts @@ -8,7 +8,10 @@ import { } from '@/agent/acp/compatibility/adapters' import { SessionTranscript } from '@/session/data/transcript' import { SessionTape } from '@/tape/application/sessionTape' -import { isEffectiveViewInputRow } from '@/tape/domain/effectiveSemantics' +import { + isEffectiveMessageInputRow, + isEffectiveViewInputRow +} from '@/tape/domain/effectiveSemantics' import type { MainDatabase } from '@/data/mainDatabase' const publishDeepchatEvent = vi.fn() @@ -188,6 +191,9 @@ function createProjectionHarness() { ), getEffectiveViewInputRows: vi.fn((sessionId: string) => tapeRows.filter((row) => row.session_id === sessionId && isEffectiveViewInputRow(row)) + ), + getEffectiveMessageInputRows: vi.fn((sessionId: string) => + tapeRows.filter((row) => row.session_id === sessionId && isEffectiveMessageInputRow(row)) ) } } as unknown as MainDatabase diff --git a/test/main/agent/deepchat/harness/deepChatAgentHarness.test.ts b/test/main/agent/deepchat/harness/deepChatAgentHarness.test.ts index aebbb1857..154092449 100644 --- a/test/main/agent/deepchat/harness/deepChatAgentHarness.test.ts +++ b/test/main/agent/deepchat/harness/deepChatAgentHarness.test.ts @@ -31,7 +31,10 @@ import { getUsableContextLength } from '@/agent/deepchat/runtime/contextBudget' import { appendMessageRecordToTape } from '@/tape/application/factPersistence' -import { isEffectiveViewInputRow } from '@/tape/domain/effectiveSemantics' +import { + isEffectiveMessageInputRow, + isEffectiveViewInputRow +} from '@/tape/domain/effectiveSemantics' import { resolveInterleavedReasoningConfig } from '@/agent/deepchat/runtime/generationSettings' import { toAcpRemoteSessionId, toAppSessionId } from '@/agent/shared/agentSessionIds' import { createLoopRun, type LoopRunRequestToolSurfaceBinding } from '@/agent/deepchat/loop/loopRun' @@ -563,6 +566,11 @@ function createMockSqlitePresenter() { (entry) => entry.session_id === sessionId && isEffectiveViewInputRow(entry) ) ), + getEffectiveMessageInputRows: vi.fn((sessionId: string) => + tapeEntries.filter( + (entry) => entry.session_id === sessionId && isEffectiveMessageInputRow(entry) + ) + ), getByEntryIds: vi.fn((sessionId: string, entryIds: readonly number[]) => { const selected = new Set(entryIds) return tapeEntries.filter( diff --git a/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts b/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts index 04be38ba7..3440a2232 100644 --- a/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts +++ b/test/main/session/data/tables/deepchatTapeEntriesTable.test.ts @@ -4,7 +4,10 @@ import { TOOL_SURFACE_TAPE_EVENT_NAMES } from '@/tape/domain/toolSurfaceFacts' import { buildTapeProviderAttemptEvent } from '@/tape/domain/providerAttempt' -import { isEffectiveViewInputRow } from '@/tape/domain/effectiveSemantics' +import { + isEffectiveMessageInputRow, + isEffectiveViewInputRow +} from '@/tape/domain/effectiveSemantics' import { TapeProviderAttemptService } from '@/tape/application/providerAttemptService' const sqliteModule = await import('better-sqlite3-multiple-ciphers').catch(() => null) @@ -815,6 +818,11 @@ describeIfSqlite('DeepChatTapeEntriesTable', () => { expect(effective.map((row) => row.entry_id)).toEqual([2, 3, 4, 5, 7]) expect(effective).toEqual(table.getBySession('s1').filter(isEffectiveViewInputRow)) + const messages = table.getEffectiveMessageInputRows('s1') + + expect(messages.map((row) => row.entry_id)).toEqual([2, 5]) + expect(messages).toEqual(table.getBySession('s1').filter(isEffectiveMessageInputRow)) + db.close() }) }) diff --git a/test/main/session/data/tapeFacts.test.ts b/test/main/session/data/tapeFacts.test.ts index 157cec7d7..56e1c9b15 100644 --- a/test/main/session/data/tapeFacts.test.ts +++ b/test/main/session/data/tapeFacts.test.ts @@ -11,7 +11,11 @@ import { } from '@/tape/domain/effectiveView' import { TAPE_COMPACTION_MODEL_CALL_EVENT_NAME } from '@/tape/domain/compactionUsage' import { TOOL_SURFACE_TAPE_EVENT_NAMES } from '@/tape/domain/toolSurfaceFacts' -import { messageRecordHasFinalToolUse, tapeToolRank } from '@/tape/domain/effectiveSemantics' +import { + isEffectiveMessageInputRow, + messageRecordHasFinalToolUse, + tapeToolRank +} from '@/tape/domain/effectiveSemantics' import type { DeepChatTapeEntryRow } from '@/tape/domain/entry' function createTable() { @@ -378,4 +382,43 @@ describe('buildEffectiveTapeView messageEntries (lineage pairing)', () => { const ids = buildEffectiveTapeView(table.rows).messageEntries.map((entry) => entry.record.id) expect(ids).toEqual(['u1']) }) + + it('derives messageRecords from message and retraction rows alone', () => { + const table = createTable() + table.append({ sessionId: 's1', kind: 'anchor', name: 'session/start', payload: {} }) + appendMessageRecordToTape(table as any, userRecord('u1', 1, 'first'), 'live') + appendMessageRecordToTape( + table as any, + assistantRecord([toolCallBlock('success', 'tc1', 'result')], { status: 'sent' }), + 'live' + ) + appendMessageRecordToTape(table as any, userRecord('u2', 3, 'retract me'), 'live') + table.append({ + sessionId: 's1', + kind: 'event', + name: 'message/retracted', + payload: { data: { messageId: 'u2' } } + }) + appendMessageRecordToTape( + table as any, + userRecord('u3', 4, 'pending', { status: 'pending' }), + 'live' + ) + table.append({ sessionId: 's1', kind: 'event', name: 'view/assembled', payload: {} }) + expect(new Set(table.rows.map((row) => row.kind))).toEqual( + new Set(['anchor', 'message', 'tool_call', 'tool_result', 'event']) + ) + + for (const includePending of [true, false]) { + const full = buildEffectiveTapeView(table.rows, { includePending }) + const messagesOnly = buildEffectiveTapeView(table.rows.filter(isEffectiveMessageInputRow), { + includePending + }) + expect(messagesOnly.messageRecords).toEqual(full.messageRecords) + expect(messagesOnly.messageEntries).toEqual(full.messageEntries) + } + expect( + buildEffectiveTapeView(table.rows, { includePending: true }).messageRecords.map((r) => r.id) + ).toEqual(['u1', 'a1', 'u3']) + }) }) diff --git a/test/main/session/data/tapeTableMockContract.test.ts b/test/main/session/data/tapeTableMockContract.test.ts index b39b2b175..021ecced0 100644 --- a/test/main/session/data/tapeTableMockContract.test.ts +++ b/test/main/session/data/tapeTableMockContract.test.ts @@ -168,7 +168,7 @@ function seedConversation(store: Store) { sessionId: SESSION, name: 'message/retracted', source: { type: 'message', id: 'm2', seq: 0 }, - data: { reason: 'user_delete' }, + data: { messageId: 'm2', reason: 'user_delete' }, createdAt: 150 }) store.appendAnchor({ @@ -258,6 +258,82 @@ function seedConversation(store: Store) { }) } +const REVISION_SESSION = 's3' + +/** + * One assistant message written three times: a sent revision that is retracted, a sent revision + * at a shifted orderSeq that supersedes it, and a pending revision that must not. Its tool rows + * use the production payload shape (`messageId`/`orderSeq` in the payload, `toolCall` as an + * object for the first call and as legacy JSON text for the second) so the join, the orderSeq + * rewrite and both `toolCall` encodings are compared between the SQL and JS effective views. + */ +function seedRevisedAssistant(store: Store) { + bootstrap(store, REVISION_SESSION, '00000000-0000-4000-8000-000000000003') + const assistant = (orderSeq: number, text: string, status: 'sent' | 'pending', seq: number) => + store.append({ + sessionId: REVISION_SESSION, + kind: 'message', + name: 'message/assistant', + source: { type: 'message', id: 'a1', seq }, + payload: { + record: createRecord({ + id: 'a1', + sessionId: REVISION_SESSION, + role: 'assistant', + orderSeq, + status, + content: JSON.stringify([{ type: 'content', content: text }]) + }) + }, + meta: { status }, + createdAt: 100 + seq + }) + const toolCall = (toolCallId: string, orderSeq: number, toolCall: unknown, createdAt: number) => + store.append({ + sessionId: REVISION_SESSION, + kind: 'tool_call', + name: 'read_file', + source: { type: 'tool_call', id: `a1:${toolCallId}`, seq: 0 }, + payload: { messageId: 'a1', orderSeq, toolCall }, + meta: { status: 'success' }, + createdAt + }) + const toolResult = (toolCallId: string, orderSeq: number, response: string, createdAt: number) => + store.append({ + sessionId: REVISION_SESSION, + kind: 'tool_result', + name: 'read_file', + source: { type: 'tool_result', id: `a1:${toolCallId}`, seq: 0 }, + payload: { messageId: 'a1', orderSeq, toolCallId, response }, + meta: { status: 'success' }, + createdAt + }) + + assistant(2, 'draft one', 'sent', 0) // entry 2 + toolCall('tc1', 2, { id: 'tc1', name: 'read_file' }, 110) // entry 3 + toolResult('tc1', 2, 'needle one', 111) // entry 4 + store.appendEvent({ + sessionId: REVISION_SESSION, + name: 'message/retracted', + source: { type: 'message', id: 'a1', seq: 0 }, + data: { messageId: 'a1', reason: 'edit' }, + createdAt: 120 + }) // entry 5 + store.append({ + sessionId: REVISION_SESSION, + kind: 'message', + name: 'message/user', + source: { type: 'message', id: 'u1', seq: 0 }, + payload: { record: createRecord({ id: 'u1', sessionId: REVISION_SESSION, orderSeq: 1 }) }, + meta: { status: 'sent' }, + createdAt: 125 + }) // entry 6 + assistant(4, 'draft two', 'sent', 1) // entry 7, orderSeq shifted from 2 to 4 + toolCall('tc2', 4, JSON.stringify({ id: 'tc2', name: 'read_file' }), 130) // entry 8 + toolResult('tc2', 4, 'needle two', 131) // entry 9 + assistant(4, 'draft three', 'pending', 2) // entry 10 +} + describe('Tape table mock contract', () => { itIfSqlite('appends rows with the same ids, provenance keys and idempotency', () => { const stores = openStores() @@ -442,6 +518,7 @@ describe('Tape table mock contract', () => { messageSourcesMissing: (store) => store.getMessageSourceEntries(SESSION, 'nope'), excludingContext: (store) => store.getBySessionExcludingContext(SESSION), effectiveViewInputs: (store) => store.getEffectiveViewInputRows(SESSION), + effectiveMessageInputs: (store) => store.getEffectiveMessageInputRows(SESSION), lineage: (store) => store.getSubagentLineageEvents(SESSION), effectiveSearchAtHeads: (store) => store.searchEffectiveSourcesAtHeads( @@ -459,13 +536,87 @@ describe('Tape table mock contract', () => { [2], { before: 1, after: 3, limit: 10 } ), - effectiveContextRetracted: (store) => + // The retracted m2 (entry 8) neither anchors a window nor appears in its neighbour's. + retractedIsNotAnAnchor: (store) => store.getEffectiveContextRowsAtHead( { sessionId: SESSION, maxEntryId: store.getMaxEntryId(SESSION) }, [8], { before: 0, after: 0, limit: 10 } + ), + retractedLeavesTheWindow: (store) => + store.getEffectiveContextRowsAtHead( + { sessionId: SESSION, maxEntryId: store.getMaxEntryId(SESSION) }, + [5], + { before: 0, after: 2, limit: 10 } ) }) + + const afterA1 = stores.real + .getEffectiveContextRowsAtHead( + { sessionId: SESSION, maxEntryId: stores.real.getMaxEntryId(SESSION) }, + [5], + { before: 0, after: 2, limit: 10 } + ) + .map((row) => row.entry_id) + .sort((left, right) => left - right) + expect(afterA1).toEqual([5, 10, 11]) + } finally { + stores.close() + } + }) + + itIfSqlite('resolves revisions, retractions and tool joins identically', () => { + const stores = openStores() + try { + for (const store of [stores.real, stores.mock]) seedRevisedAssistant(store) + + const head = (store: Store) => ({ + sessionId: REVISION_SESSION, + maxEntryId: store.getMaxEntryId(REVISION_SESSION) + }) + expectParity(stores, { + // The surviving a1 revision (entry 7) with every tool row joined onto it, each tool row's + // stored orderSeq rewritten to the revision's orderSeq. + effectiveWindow: (store) => + store.getEffectiveContextRowsAtHead(head(store), [7], { before: 5, after: 5, limit: 20 }), + // The retracted first revision is not an effective row, so nothing anchors the window. + retractedRevision: (store) => + store.getEffectiveContextRowsAtHead(head(store), [2], { before: 1, after: 1, limit: 20 }), + // Before the retraction the first revision and its object-form tool call are effective. + firstRevisionWindow: (store) => + store.getEffectiveContextRowsAtHead({ sessionId: REVISION_SESSION, maxEntryId: 4 }, [2], { + before: 1, + after: 3, + limit: 20 + }), + // Tool results of both revisions match; the pending third revision contributes nothing. + toolResultSearch: (store) => store.searchEffectiveSourcesAtHeads([head(store)], 'needle'), + pendingSearch: (store) => store.searchEffectiveSourcesAtHeads([head(store)], 'draft three') + }) + + // The reader lists requested rows before their neighbours; sort to assert the window's content. + const window = stores.real + .getEffectiveContextRowsAtHead(head(stores.real), [7], { before: 5, after: 5, limit: 20 }) + .sort((left, right) => left.entry_id - right.entry_id) + expect(window.map((row) => [row.entry_id, row.kind])).toEqual([ + [1, 'anchor'], + [3, 'tool_call'], + [4, 'tool_result'], + [6, 'message'], + [7, 'message'], + [8, 'tool_call'], + [9, 'tool_result'] + ]) + expect( + window + .filter((row) => row.kind === 'tool_call' || row.kind === 'tool_result') + .map((row) => JSON.parse(row.payload_json).orderSeq) + ).toEqual([4, 4, 4, 4]) + expect( + stores.real + .searchEffectiveSourcesAtHeads([head(stores.real)], 'needle') + .map((row) => row.entry_id) + ).toEqual([9, 4]) } finally { stores.close() } diff --git a/test/main/session/data/tapeTestHarness.ts b/test/main/session/data/tapeTestHarness.ts index 21a886e52..6cf78790a 100644 --- a/test/main/session/data/tapeTestHarness.ts +++ b/test/main/session/data/tapeTestHarness.ts @@ -20,7 +20,10 @@ import { DeepChatTapeEntriesTable, MAX_TAPE_SEARCH_TOKEN_CLAUSES } from '@/tape/infrastructure/sqlite/tapeEntryStore' -import { isEffectiveViewInputRow } from '@/tape/domain/effectiveSemantics' +import { + isEffectiveMessageInputRow, + isEffectiveViewInputRow +} from '@/tape/domain/effectiveSemantics' import { SUMMARY_ANCHOR_NAMES } from '@/tape/domain/entry' import { EXECUTION_JOURNAL_EVENT_NAMES } from '@/tape/domain/executionJournal' import { @@ -440,6 +443,9 @@ function createTapeTableMock() { getEffectiveViewInputRows: vi.fn((sessionId: string) => entries.filter((entry) => entry.session_id === sessionId && isEffectiveViewInputRow(entry)) ), + getEffectiveMessageInputRows: vi.fn((sessionId: string) => + entries.filter((entry) => entry.session_id === sessionId && isEffectiveMessageInputRow(entry)) + ), getByEntryIds: vi.fn((sessionId: string, entryIds: readonly number[]) => { const selected = new Set(entryIds) return entries.filter( diff --git a/test/main/session/runtimeIntegration.test.ts b/test/main/session/runtimeIntegration.test.ts index 7b21d50b7..b7c9aff41 100644 --- a/test/main/session/runtimeIntegration.test.ts +++ b/test/main/session/runtimeIntegration.test.ts @@ -14,7 +14,10 @@ import { createSessionQueryFixture } from './queryFixture' import { createSessionFixture } from './sessionFixture' import { createSessionData, createSessionDataFromDatabase } from '@/session/data' import { SessionTranscriptMutations } from '@/session/transcriptMutations' -import { isEffectiveViewInputRow } from '@/tape/domain/effectiveSemantics' +import { + isEffectiveMessageInputRow, + isEffectiveViewInputRow +} from '@/tape/domain/effectiveSemantics' import { createPassthroughModelRequestPolicy } from '@shared/modelRequestPolicy' import { POSIX_COMMAND_SHELL } from '../../helpers/commandShell' @@ -179,6 +182,11 @@ function createMockSqlitePresenter() { (entry) => entry.session_id === sessionId && isEffectiveViewInputRow(entry) ) ), + getEffectiveMessageInputRows: vi.fn((sessionId: string) => + tapeEntries.filter( + (entry) => entry.session_id === sessionId && isEffectiveMessageInputRow(entry) + ) + ), getByEntryIds: vi.fn((sessionId: string, entryIds: readonly number[]) => { const requestedIds = new Set(entryIds) return tapeEntries.filter( diff --git a/test/main/tape/executionJournal.test.ts b/test/main/tape/executionJournal.test.ts index ddaa96ab3..b5d87d90f 100644 --- a/test/main/tape/executionJournal.test.ts +++ b/test/main/tape/executionJournal.test.ts @@ -848,6 +848,32 @@ describe('Execution Journal domain and strict persistence', () => { ]) }) + it('canonicalises the Run ID and only accepts UUID versions 1-8', () => { + const { table, entries } = createTapeTableMock() + const service = createTapeService(table) + + // Hex letters, so the upper-case spelling really differs from the canonical one. + const canonical = 'abcdef01-2345-4678-8abc-def012345678' + const upperCase = canonical.toUpperCase() + expect(upperCase).not.toBe(canonical) + + const first = commitStarted(service, upperCase) + const replay = commitStarted(service, canonical) + expect(first.created).toBe(true) + expect(replay).toEqual({ ...first, created: false }) + expect(JSON.parse(entries[entries.length - 1].payload_json).data.runId).toBe(canonical) + expect(entries[entries.length - 1].provenance_key).toBe( + buildExecutionRunProvenanceKey(canonical, 'started') + ) + + for (const version of ['0', '9', 'f']) { + expect(() => commitStarted(service, `55555555-5555-${version}555-8555-555555555555`)).toThrow( + /runId must be a UUID/ + ) + } + expect(entries.filter((entry) => entry.name?.startsWith('execution/'))).toHaveLength(1) + }) + it('rejects non-JSON arguments and invalid timestamps before append', () => { const { table, entries } = createTapeTableMock() const service = createTapeService(table)