diff --git a/action.yml b/action.yml index 410a5d3c3..01962881c 100644 --- a/action.yml +++ b/action.yml @@ -19,7 +19,10 @@ inputs: required: false default: 'run' findings-file: - description: 'Path to structured JSON findings file to read in report mode.' + description: 'Path to structured JSON findings file to read in report mode. When output-schema-version is "2", this is the schema-v2 findings file.' + required: false + metadata-file: + description: 'Path to schema-v2 metadata JSON file to read in report mode, required alongside findings-file when output-schema-version is "2".' required: false base-config-path: description: 'Path to the org-wide base warden.toml file to load before the repo config (relative to repo root)' @@ -55,6 +58,13 @@ inputs: description: 'Maximum number of concurrent trigger executions' required: false default: '5' + output-schema-version: + description: 'Output schema for findings-file. "2" additionally writes warden-metadata.json and warden-findings-v2.json.' + required: false + default: '1' + action-ref: + description: 'Pinned SHA/ref of this action as referenced by the calling workflow, recorded in schema v2 metadata as harness.actionRef' + required: false outputs: findings-count: @@ -67,8 +77,14 @@ outputs: description: 'Summary of the analysis' value: ${{ steps.warden.outputs.summary }} findings-file: - description: 'Path to structured JSON findings file (always written, use for upload to GCS/S3/etc.)' + description: 'Path to structured JSON findings file (always written, use for upload to GCS/S3/etc.). When output-schema-version is "2", this is the schema-v2 findings file, matching the findings-file input.' value: ${{ steps.warden.outputs.findings-file }} + metadata-file: + description: 'Path to schema-v2 metadata JSON file (only written when output-schema-version is "2")' + value: ${{ steps.warden.outputs.metadata-file }} + findings-file-v2: + description: 'Path to schema-v2 findings JSON file (only written when output-schema-version is "2")' + value: ${{ steps.warden.outputs.findings-file-v2 }} runs: using: 'composite' @@ -81,6 +97,7 @@ runs: INPUT_GITHUB_TOKEN: ${{ inputs.github-token }} INPUT_MODE: ${{ inputs.mode }} INPUT_FINDINGS_FILE: ${{ inputs.findings-file }} + INPUT_METADATA_FILE: ${{ inputs.metadata-file }} INPUT_BASE_CONFIG_PATH: ${{ inputs.base-config-path }} INPUT_BASE_SKILL_ROOT: ${{ inputs.base-skill-root }} INPUT_CONFIG_PATH: ${{ inputs.config-path }} @@ -90,4 +107,6 @@ runs: INPUT_REQUEST_CHANGES: ${{ inputs.request-changes }} INPUT_FAIL_CHECK: ${{ inputs.fail-check }} INPUT_PARALLEL: ${{ inputs.parallel }} + INPUT_OUTPUT_SCHEMA_VERSION: ${{ inputs.output-schema-version }} + INPUT_ACTION_REF: ${{ inputs.action-ref }} run: node ${{ github.action_path }}/dist/action/index.js diff --git a/packages/warden/src/action/inputs.test.ts b/packages/warden/src/action/inputs.test.ts index 0d29906e4..b0b5a7c9a 100644 --- a/packages/warden/src/action/inputs.test.ts +++ b/packages/warden/src/action/inputs.test.ts @@ -155,6 +155,28 @@ describe('parseActionInputs', () => { expect(() => parseActionInputs()).toThrow('Invalid mode "later"'); }); }); + + describe('output-schema-version handling', () => { + it('defaults to schema version 1', () => { + const inputs = parseActionInputs(); + expect(inputs.outputSchemaVersion).toBe('1'); + }); + + it('parses output-schema-version 2, metadata-file, and action-ref', () => { + process.env['INPUT_OUTPUT_SCHEMA_VERSION'] = '2'; + process.env['INPUT_METADATA_FILE'] = 'warden-metadata.json'; + process.env['INPUT_ACTION_REF'] = 'abc123'; + const inputs = parseActionInputs(); + expect(inputs.outputSchemaVersion).toBe('2'); + expect(inputs.metadataFile).toBe('warden-metadata.json'); + expect(inputs.actionRef).toBe('abc123'); + }); + + it('rejects an invalid output-schema-version', () => { + process.env['INPUT_OUTPUT_SCHEMA_VERSION'] = '3'; + expect(() => parseActionInputs()).toThrow('Invalid output-schema-version "3"'); + }); + }); }); describe('setupAuthEnv', () => { @@ -173,6 +195,7 @@ describe('setupAuthEnv', () => { configPath: 'warden.toml', maxFindings: 50, parallel: 4, + outputSchemaVersion: '1', }); expect(process.env['ANTHROPIC_API_KEY']).toBe('sk-ant-api-key'); expect(process.env['WARDEN_ANTHROPIC_API_KEY']).toBe('sk-ant-api-key'); @@ -188,6 +211,7 @@ describe('setupAuthEnv', () => { configPath: 'warden.toml', maxFindings: 50, parallel: 4, + outputSchemaVersion: '1', }); expect(process.env['CLAUDE_CODE_OAUTH_TOKEN']).toBe('sk-ant-oat-oauth-token'); expect(process.env['ANTHROPIC_API_KEY']).toBeUndefined(); @@ -206,6 +230,7 @@ describe('setupAuthEnv', () => { configPath: 'warden.toml', maxFindings: 50, parallel: 4, + outputSchemaVersion: '1', }); expect(process.env['CLAUDE_CODE_OAUTH_TOKEN']).toBe('sk-ant-oat-oauth-token'); @@ -226,6 +251,7 @@ describe('setupAuthEnv', () => { configPath: 'warden.toml', maxFindings: 50, parallel: 4, + outputSchemaVersion: '1', }); expect(process.env['CLAUDE_CODE_OAUTH_TOKEN']).toBeUndefined(); @@ -245,6 +271,7 @@ describe('validateInputs', () => { configPath: 'warden.toml', maxFindings: 50, parallel: 4, + outputSchemaVersion: '1', })).toThrow('base-skill-root requires base-config-path'); }); @@ -257,6 +284,35 @@ describe('validateInputs', () => { configPath: 'warden.toml', maxFindings: 50, parallel: 4, + outputSchemaVersion: '1', })).toThrow('findings-file is required when mode is report'); }); + + it('requires metadata-file in report mode when output-schema-version is 2', () => { + expect(() => validateInputs({ + anthropicApiKey: 'sk-ant-api-key', + oauthToken: '', + githubToken: 'test', + mode: 'report', + configPath: 'warden.toml', + maxFindings: 50, + parallel: 4, + outputSchemaVersion: '2', + findingsFile: 'warden-findings-v2.json', + })).toThrow("metadata-file is required when mode is report and output-schema-version is '2'"); + }); + + it('does not require metadata-file in report mode when output-schema-version is 1', () => { + expect(() => validateInputs({ + anthropicApiKey: 'sk-ant-api-key', + oauthToken: '', + githubToken: 'test', + mode: 'report', + configPath: 'warden.toml', + maxFindings: 50, + parallel: 4, + outputSchemaVersion: '1', + findingsFile: 'warden-findings.json', + })).not.toThrow(); + }); }); diff --git a/packages/warden/src/action/inputs.ts b/packages/warden/src/action/inputs.ts index da7b43a93..332145320 100644 --- a/packages/warden/src/action/inputs.ts +++ b/packages/warden/src/action/inputs.ts @@ -24,6 +24,8 @@ export interface ActionInputs { mode: ActionMode; /** Structured findings file used by report mode */ findingsFile?: string; + /** Schema-v2 metadata file used by report mode when outputSchemaVersion is '2' */ + metadataFile?: string; /** Optional org-wide base config that is loaded before the repo config */ baseConfigPath?: string; /** Optional repo root containing org-shared local skills for the base config */ @@ -39,6 +41,10 @@ export interface ActionInputs { failCheck?: boolean; /** Max concurrent trigger executions */ parallel: number; + /** Output schema for the written artifacts. '2' additionally writes warden-metadata.json and warden-findings-v2.json. */ + outputSchemaVersion: '1' | '2'; + /** Pinned SHA/ref of the calling workflow's `uses:` line, for harness.actionRef in schema v2. */ + actionRef?: string; } // ----------------------------------------------------------------------------- @@ -77,6 +83,14 @@ function parseModeInput(value: string): ActionMode { throw new Error(`Invalid mode "${mode}". Expected run, analyze, or report.`); } +function parseOutputSchemaVersionInput(value: string): '1' | '2' { + const version = value || '1'; + if (version === '1' || version === '2') { + return version; + } + throw new Error(`Invalid output-schema-version "${version}". Expected 1 or 2.`); +} + /** * Parse action inputs from the GitHub Actions environment. * Runtime-specific auth can be absent here; runtime setup validates it when needed. @@ -112,12 +126,15 @@ export function parseActionInputs(): ActionInputs { const requestChanges = parseBooleanInput(getInput('request-changes')); const failCheck = parseBooleanInput(getInput('fail-check')); + const outputSchemaVersion = parseOutputSchemaVersionInput(getInput('output-schema-version')); + return { anthropicApiKey, oauthToken, githubToken: getInput('github-token') || process.env['GITHUB_TOKEN'] || '', mode: parseModeInput(getInput('mode')), findingsFile: getInput('findings-file') || undefined, + metadataFile: getInput('metadata-file') || undefined, baseConfigPath: getInput('base-config-path') || undefined, baseSkillRoot: getInput('base-skill-root') || undefined, configPath: getInput('config-path') || 'warden.toml', @@ -127,6 +144,8 @@ export function parseActionInputs(): ActionInputs { requestChanges, failCheck, parallel: Number.isNaN(parallelParsed) ? DEFAULT_CONCURRENCY : parallelParsed, + outputSchemaVersion, + actionRef: getInput('action-ref') || undefined, }; } @@ -144,6 +163,9 @@ export function validateInputs(inputs: ActionInputs): void { if (inputs.mode === 'report' && !inputs.findingsFile) { throw new Error('findings-file is required when mode is report'); } + if (inputs.mode === 'report' && inputs.outputSchemaVersion === '2' && !inputs.metadataFile) { + throw new Error('metadata-file is required when mode is report and output-schema-version is \'2\''); + } } /** diff --git a/packages/warden/src/action/reporting/outcomes.ts b/packages/warden/src/action/reporting/outcomes.ts index 2250ac085..baadae3eb 100644 --- a/packages/warden/src/action/reporting/outcomes.ts +++ b/packages/warden/src/action/reporting/outcomes.ts @@ -11,16 +11,18 @@ export type FindingOutcome = export type DedupeSource = 'warden' | 'external'; export type DedupeMatchType = 'hash' | 'semantic'; -export type SkippedReason = 'max_findings' | 'duplicate_in_batch' | 'no_inline_location'; +export type SkippedReason = 'max_findings' | 'duplicate_in_batch' | 'no_inline_location' | 'review_not_posted'; export type ResolvedReason = 'fix_evaluation' | 'stale_check'; export const DedupeDetailSchema = z.object({ source: z.enum(['warden', 'external']), matchType: z.enum(['hash', 'semantic']), existingFindingId: z.string().optional(), + existingSkillExecutionId: z.string().optional(), existingCommentId: z.number().int().positive().optional(), existingThreadId: z.string().optional(), existingResolved: z.boolean().optional(), + existingSkills: z.array(z.string()).optional(), actor: z.string().optional(), }); @@ -29,10 +31,13 @@ export type DedupeDetail = z.infer; interface BaseFindingObservation { finding: Finding; skill?: string; + skillExecutionId?: string; } export interface PostedFindingObservation extends BaseFindingObservation { outcome: 'posted'; + githubCommentId?: number; + githubCommentUrl?: string; } export interface DedupedFindingObservation extends BaseFindingObservation { @@ -66,29 +71,36 @@ export const FindingObservationSchema = z.discriminatedUnion('outcome', [ outcome: z.literal('posted'), finding: FindingSchema, skill: z.string().optional(), + skillExecutionId: z.string().optional(), + githubCommentId: z.number().int().positive().optional(), + githubCommentUrl: z.string().optional(), }), z.object({ outcome: z.literal('deduped'), finding: FindingSchema, skill: z.string().optional(), + skillExecutionId: z.string().optional(), dedupe: DedupeDetailSchema, }), z.object({ outcome: z.literal('skipped'), finding: FindingSchema, skill: z.string().optional(), - skippedReason: z.enum(['max_findings', 'duplicate_in_batch', 'no_inline_location']), + skillExecutionId: z.string().optional(), + skippedReason: z.enum(['max_findings', 'duplicate_in_batch', 'no_inline_location', 'review_not_posted']), }), z.object({ outcome: z.literal('resolved'), finding: FindingSchema, skill: z.string().optional(), + skillExecutionId: z.string().optional(), resolvedReason: z.enum(['fix_evaluation', 'stale_check']), }), z.object({ outcome: z.literal('failed'), finding: FindingSchema, skill: z.string().optional(), + skillExecutionId: z.string().optional(), }), ]); diff --git a/packages/warden/src/action/reporting/output-v2.test.ts b/packages/warden/src/action/reporting/output-v2.test.ts new file mode 100644 index 000000000..6c75f299b --- /dev/null +++ b/packages/warden/src/action/reporting/output-v2.test.ts @@ -0,0 +1,1497 @@ +import { describe, expect, it } from 'vitest'; +import type { EventContext, Finding, SkillReport } from '../../types/index.js'; +import type { ResolvedTrigger } from '../../config/loader.js'; +import type { TriggerResult } from '../triggers/executor.js'; +import type { FindingObservation } from './outcomes.js'; +import { + buildFindingsOutputV2, + buildMetadataOutputV2, + fromAuxiliaryUsageEntries, + patchFindingsOutputV2Observations, + reconstructSkillReportsFromV2, + WardenFindingsSchemaV2, + WardenMetadataSchema, +} from './output-v2.js'; + +function createContext(overrides: Partial = {}): EventContext { + return { + eventType: 'pull_request', + action: 'opened', + repository: { + owner: 'getsentry', + name: 'warden', + fullName: 'getsentry/warden', + defaultBranch: 'main', + }, + pullRequest: { + number: 362, + title: 'Test PR', + body: '', + author: 'user-123', + baseBranch: 'main', + headBranch: 'feature', + headSha: 'abc123', + baseSha: 'def456', + files: [], + }, + repoPath: '/repo', + ...overrides, + }; +} + +function createTrigger(overrides: Partial = {}): ResolvedTrigger { + return { + id: 'trigger-id', + skillExecutionId: 'exec-1', + name: 'test-trigger', + skill: 'code-review', + type: 'pull_request', + actions: ['opened'], + filters: {}, + ...overrides, + }; +} + +function createFinding(overrides: Partial = {}): Finding { + return { + id: 'WRD-001', + severity: 'high', + confidence: 'high', + title: 'Finding title', + description: 'Finding description', + location: { path: 'src/index.ts', startLine: 1 }, + ...overrides, + }; +} + +function createReport(overrides: Partial = {}): SkillReport { + return { + skill: 'code-review', + summary: 'Found 1 issue', + findings: [createFinding()], + model: 'claude-sonnet-5', + ...overrides, + }; +} + +function createResult(overrides: Partial = {}): TriggerResult { + return { + triggerId: 'trigger-id', + triggerName: 'test-trigger', + skillName: 'code-review', + skillExecutionId: 'exec-1', + report: createReport(), + ...overrides, + }; +} + +describe('buildMetadataOutputV2', () => { + it('builds a schema-valid metadata payload', () => { + const matched = createTrigger(); + const skipped = createTrigger({ + id: 'skipped-id', + skillExecutionId: 'exec-2', + name: 'skipped-trigger', + skill: 'security-review', + actions: ['synchronize'], + }); + + const output = buildMetadataOutputV2( + createContext(), + [matched, skipped], + [matched], + [createResult()], + { runId: '123', generatedAt: '2026-01-01T00:00:00.000Z' } + ); + + expect(WardenMetadataSchema.parse(output)).toEqual(output); + expect(output.harness.name).toBe('warden'); + expect(output.skippedTriggers).toEqual([ + { skillName: 'security-review', triggerId: 'skipped-id', triggerName: 'skipped-trigger', reason: 'no_event_match' }, + ]); + }); + + it('reports label_mismatch, not draft_state, when a labeled event adds a non-matching label', () => { + const matched = createTrigger(); + const skipped = createTrigger({ + id: 'skipped-id', + skillExecutionId: 'exec-2', + name: 'skipped-trigger', + skill: 'security-review', + actions: ['labeled'], + labels: ['deploy-ready'], + }); + + const context = createContext({ + action: 'labeled', + label: 'wont-fix', + pullRequest: { + number: 4821, + title: 'Test PR', + body: '', + author: 'octocat', + baseBranch: 'main', + headBranch: 'feature', + headSha: 'abc123', + baseSha: 'def456', + files: [], + labels: ['deploy-ready', 'wont-fix'], + }, + }); + + const output = buildMetadataOutputV2( + context, + [matched, skipped], + [matched], + [createResult()], + { runId: '123', generatedAt: '2026-01-01T00:00:00.000Z' } + ); + + expect(output.skippedTriggers).toEqual([ + { skillName: 'security-review', triggerId: 'skipped-id', triggerName: 'skipped-trigger', reason: 'label_mismatch' }, + ]); + }); + + it('reports path_filter when the event and state match but no changed file satisfies the path filter', () => { + const matched = createTrigger(); + const skipped = createTrigger({ + id: 'skipped-id', + skillExecutionId: 'exec-2', + name: 'skipped-trigger', + skill: 'security-review', + filters: { paths: ['src/**'] }, + }); + + const output = buildMetadataOutputV2( + createContext(), + [matched, skipped], + [matched], + [createResult()], + { runId: '123', generatedAt: '2026-01-01T00:00:00.000Z' } + ); + + expect(output.skippedTriggers).toEqual([ + { skillName: 'security-review', triggerId: 'skipped-id', triggerName: 'skipped-trigger', reason: 'path_filter' }, + ]); + }); + + it('reports no_changes when a schedule trigger fires with no changed files to scan', () => { + const matched = createTrigger(); + const skipped = createTrigger({ + id: 'skipped-id', + skillExecutionId: 'exec-2', + name: 'skipped-trigger', + skill: 'security-review', + type: 'schedule', + }); + + const output = buildMetadataOutputV2( + createContext({ eventType: 'schedule' }), + [matched, skipped], + [matched], + [createResult()], + { runId: '123', generatedAt: '2026-01-01T00:00:00.000Z' } + ); + + expect(output.skippedTriggers).toEqual([ + { skillName: 'security-review', triggerId: 'skipped-id', triggerName: 'skipped-trigger', reason: 'no_changes' }, + ]); + }); + + it('reports no_event_match, not path_filter, for a wildcard trigger skipped on a schedule run', () => { + // schedule.ts only ever evaluates type: 'schedule' triggers - a wildcard + // trigger never reaches a real path-filter check in a scheduled run, so + // it's excluded by event type, not by its (possibly nonexistent) paths. + const matched = createTrigger({ type: 'schedule' }); + const skipped = createTrigger({ + id: 'skipped-id', + skillExecutionId: 'exec-2', + name: 'skipped-trigger', + skill: 'security-review', + type: '*', + }); + + const output = buildMetadataOutputV2( + createContext({ eventType: 'schedule' }), + [matched, skipped], + [matched], + [createResult()], + { runId: '123', generatedAt: '2026-01-01T00:00:00.000Z' } + ); + + expect(output.skippedTriggers).toEqual([ + { skillName: 'security-review', triggerId: 'skipped-id', triggerName: 'skipped-trigger', reason: 'no_event_match' }, + ]); + }); + + it('falls back to the action-level failOn/reportOn when the primary trigger has no override', () => { + const matched = createTrigger({ failOn: undefined, reportOn: undefined }); + + const output = buildMetadataOutputV2( + createContext(), + [matched], + [matched], + [createResult()], + { runId: '123', generatedAt: '2026-01-01T00:00:00.000Z', failOn: 'high', reportOn: 'medium' } + ); + + expect(output.resolvedDefaults).toEqual( + expect.objectContaining({ failOn: 'high', reportOn: 'medium' }) + ); + }); + + it('falls back to the executor\'s own "medium" default when the primary trigger has no minConfidence override', () => { + // executor.ts applies `trigger.minConfidence ?? 'medium'` at runtime, so + // an unconfigured trigger still runs at 'medium' - resolvedDefaults must + // mirror that, not report minConfidence as unset. + const matched = createTrigger({ minConfidence: undefined }); + + const output = buildMetadataOutputV2( + createContext(), + [matched], + [matched], + [createResult()], + { runId: '123', generatedAt: '2026-01-01T00:00:00.000Z' } + ); + + expect(output.resolvedDefaults?.minConfidence).toBe('medium'); + }); + + it('resolves failCheck/requestChanges/maxFindings the same way as failOn/reportOn', () => { + const matched = createTrigger({ failCheck: undefined, requestChanges: undefined, maxFindings: undefined }); + + const output = buildMetadataOutputV2( + createContext(), + [matched], + [matched], + [createResult()], + { + runId: '123', generatedAt: '2026-01-01T00:00:00.000Z', + failCheck: true, requestChanges: true, maxFindings: 10, + } + ); + + expect(output.resolvedDefaults).toEqual( + expect.objectContaining({ failCheck: true, requestChanges: true, maxFindings: 10 }) + ); + }); + + it('prefers the primary trigger override over the action-level failCheck/requestChanges/maxFindings', () => { + const matched = createTrigger({ failCheck: false, requestChanges: false, maxFindings: 5 }); + + const output = buildMetadataOutputV2( + createContext(), + [matched], + [matched], + [createResult()], + { + runId: '123', generatedAt: '2026-01-01T00:00:00.000Z', + failCheck: true, requestChanges: true, maxFindings: 10, + } + ); + + expect(output.resolvedDefaults).toEqual( + expect.objectContaining({ failCheck: false, requestChanges: false, maxFindings: 5 }) + ); + }); +}); + +describe('buildFindingsOutputV2', () => { + it('builds a schema-valid findings payload with primary attribution', () => { + const trigger = createTrigger(); + const output = buildFindingsOutputV2([createResult()], [trigger], [], { runId: '123' }); + + expect(WardenFindingsSchemaV2.parse(output)).toEqual(output); + expect(output.findings).toHaveLength(1); + expect(output.findings[0]?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + ]); + expect(output.skillExecutions[0]?.findingsBySeverity).toEqual({ high: 1, medium: 0, low: 0 }); + }); + + it('carries the check run url/id from the trigger result onto the skill execution', () => { + const trigger = createTrigger(); + const result = createResult({ checkRunUrl: 'https://github.com/getsentry/warden/runs/999', checkRunId: 999 }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + expect(output.skillExecutions[0]?.checkRunUrl).toBe('https://github.com/getsentry/warden/runs/999'); + expect(output.skillExecutions[0]?.checkRunId).toBe(999); + }); + + it('exports the already-computed review event and check conclusion per execution', () => { + const trigger = createTrigger(); + const result = createResult({ + report: createReport({ findings: [createFinding({ severity: 'high' })] }), + failOn: 'high', + failCheck: true, + renderResult: { review: { event: 'REQUEST_CHANGES', body: '', comments: [] }, summaryComment: '' }, + }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + expect(output.skillExecutions[0]?.reviewEvent).toBe('REQUEST_CHANGES'); + expect(output.skillExecutions[0]?.checkConclusion).toBe('failure'); + }); + + it('carries the schedule-created GitHub issue number/url onto the skill execution', () => { + const trigger = createTrigger(); + const result = createResult({ + issueNumber: 42, + issueUrl: 'https://github.com/getsentry/warden/issues/42', + }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + expect(output.skillExecutions[0]?.issueNumber).toBe(42); + expect(output.skillExecutions[0]?.issueUrl).toBe('https://github.com/getsentry/warden/issues/42'); + }); + + it('leaves reviewEvent unset and reports a success conclusion when there are no findings', () => { + const trigger = createTrigger(); + const result = createResult({ report: createReport({ findings: [] }), failOn: 'high', failCheck: true }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + expect(output.skillExecutions[0]?.reviewEvent).toBeUndefined(); + expect(output.skillExecutions[0]?.checkConclusion).toBe('success'); + }); + + it('reports the observed model on the skill execution and finding provenance, distinct from the configured model', () => { + const trigger = createTrigger({ model: 'claude-opus-4-5' }); + const result = createResult({ report: createReport({ model: 'claude-haiku-4-5' }) }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + expect(output.skillExecutions[0]?.model).toBe('claude-haiku-4-5'); + expect(output.findings[0]?.provenance.originModel).toBe('claude-haiku-4-5'); + + const metadata = buildMetadataOutputV2(createContext(), [trigger], [trigger], [result], { + runId: '123', + generatedAt: '2026-01-01T00:00:00.000Z', + }); + expect(metadata.resolvedDefaults?.model).toBe('claude-opus-4-5'); + }); + + it('round-trips the disagreement-visibility models[] field onto the skill execution', () => { + const trigger = createTrigger(); + const result = createResult({ + report: createReport({ model: 'claude-opus-4-5', models: ['claude-haiku-4-5', 'claude-sonnet-5'] }), + }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + expect(output.skillExecutions[0]?.model).toBe('claude-opus-4-5'); + expect(output.skillExecutions[0]?.models).toEqual(['claude-haiku-4-5', 'claude-sonnet-5']); + + const reconstructed = reconstructSkillReportsFromV2(output); + expect(reconstructed[0]?.models).toEqual(['claude-haiku-4-5', 'claude-sonnet-5']); + }); + + it('computes checkConclusion from confidence-filtered findings, matching the real GitHub check', () => { + const trigger = createTrigger(); + const result = createResult({ + report: createReport({ findings: [createFinding({ severity: 'high', confidence: 'low' })] }), + failOn: 'high', + failCheck: true, + minConfidence: 'high', + }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + // buildSkillCheckPayload (github-checks.ts) filters by minConfidence before + // computing conclusion, so a low-confidence finding filtered out by + // minConfidence: 'high' must not count toward failure here either. + expect(output.skillExecutions[0]?.checkConclusion).toBe('success'); + }); + + it('attaches githubCommentId/githubCommentUrl from a posted finding observation', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-501' }); + const observations: FindingObservation[] = [ + { + outcome: 'posted', + finding, + skill: 'code-review', + skillExecutionId: 'exec-1', + githubCommentId: 42, + githubCommentUrl: 'https://github.com/getsentry/warden/pull/1#discussion_r42', + }, + ]; + + const output = buildFindingsOutputV2( + [createResult({ report: createReport({ findings: [finding] }) })], + [trigger], + observations, + { runId: '123' } + ); + + expect(output.findings[0]?.githubCommentId).toBe(42); + expect(output.findings[0]?.githubCommentUrl).toBe('https://github.com/getsentry/warden/pull/1#discussion_r42'); + }); + + it('records verifier revisions in per-finding provenance', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-002', severity: 'medium' }); + const result = createResult({ + report: createReport({ findings: [finding] }), + findingProcessingEvents: [ + { + stage: 'verification', + action: 'revised', + finding: createFinding({ id: 'WRD-002', severity: 'high', title: 'Original title' }), + replacement: finding, + reason: 'narrower scope', + model: 'claude-haiku-4-5', + }, + ], + }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + expect(output.findings[0]?.provenance.verification).toEqual({ + outcome: 'revised', + model: 'claude-haiku-4-5', + runtime: undefined, + evidence: undefined, + before: { + title: 'Original title', + description: 'Finding description', + severity: 'high', + confidence: 'high', + }, + }); + }); + + it('does not leak verification provenance across skills when finding ids collide', () => { + const firstTrigger = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const secondTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const revisedFinding = createFinding({ id: 'WRD-001', severity: 'medium' }); + const firstResult = createResult({ + triggerId: 'trigger-1', + report: createReport({ skill: 'code-review', findings: [revisedFinding] }), + findingProcessingEvents: [ + { + stage: 'verification', + action: 'revised', + finding: createFinding({ id: 'WRD-001', severity: 'high', title: 'Original title' }), + replacement: revisedFinding, + reason: 'narrower scope', + model: 'claude-haiku-4-5', + }, + ], + }); + + const unrelatedFinding = createFinding({ id: 'WRD-001', title: 'Unrelated finding, same id' }); + const secondResult = createResult({ + triggerId: 'trigger-2', + report: createReport({ skill: 'security-review', findings: [unrelatedFinding] }), + }); + + const output = buildFindingsOutputV2( + [firstResult, secondResult], + [firstTrigger, secondTrigger], + [], + { runId: '123' } + ); + + const fromSecondSkill = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-2'); + expect(fromSecondSkill?.provenance.verification).toBeUndefined(); + }); + + it('does not attach corroboration to an unrelated finding that happens to share an id', () => { + const firstTrigger = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const secondTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'unrelated-skill-trigger', + skill: 'unrelated-skill', + }); + const thirdTrigger = createTrigger({ + id: 'trigger-3', + skillExecutionId: 'exec-3', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const firstResult = createResult({ + triggerId: 'trigger-1', + report: createReport({ skill: 'code-review', findings: [createFinding({ id: 'WRD-001' })] }), + }); + const secondResult = createResult({ + triggerId: 'trigger-2', + report: createReport({ skill: 'unrelated-skill', findings: [createFinding({ id: 'WRD-001' })] }), + }); + + // The dedupe match names its skill(s) explicitly, so this corroboration + // targets code-review's WRD-001, not unrelated-skill's same-id finding. + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-004' }), + skill: 'security-review', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-001', + existingSkills: ['code-review'], + }, + }, + ]; + + const output = buildFindingsOutputV2( + [firstResult, secondResult], + [firstTrigger, secondTrigger, thirdTrigger], + observations, + { runId: '123' } + ); + + const fromCodeReview = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-1'); + const fromUnrelated = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-2'); + + expect(fromCodeReview?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + { skillExecutionId: 'exec-3', skillName: 'security-review', role: 'corroborating', matchType: 'hash' }, + ]); + expect(fromUnrelated?.reportedBy).toEqual([ + { skillExecutionId: 'exec-2', skillName: 'unrelated-skill', role: 'primary' }, + ]); + }); + + it('does not let a second own-finding anchor overwrite the first when two findings from one execution share a bare prior id', () => { + const primaryTrigger = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const corroboratingTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const primaryResult = createResult({ + triggerId: 'trigger-1', + report: createReport({ + skill: 'code-review', + findings: [ + createFinding({ id: 'WRD-001', reportedId: 'WRD-777' }), + createFinding({ id: 'WRD-002', reportedId: 'WRD-777' }), + ], + }), + }); + + // Both of code-review's own findings dedupe against prior comments that + // happen to share the same bare external id - only the first (comment + // 100) is truly corroborated by security-review; the second (comment + // 200) is not. + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-001' }), + skill: 'code-review', + skillExecutionId: 'exec-1', + dedupe: { source: 'warden', matchType: 'hash', existingFindingId: 'WRD-777', existingCommentId: 100 }, + }, + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-002' }), + skill: 'code-review', + skillExecutionId: 'exec-1', + dedupe: { source: 'warden', matchType: 'hash', existingFindingId: 'WRD-777', existingCommentId: 200 }, + }, + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-099' }), + skill: 'security-review', + skillExecutionId: 'exec-2', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-777', + existingCommentId: 100, + existingSkills: ['code-review'], + }, + }, + ]; + + const output = buildFindingsOutputV2( + [primaryResult], + [primaryTrigger, corroboratingTrigger], + observations, + { runId: '123' } + ); + + const f1 = output.findings.find((f) => f.id === 'WRD-001'); + const f2 = output.findings.find((f) => f.id === 'WRD-002'); + + expect(f1?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + { skillExecutionId: 'exec-2', skillName: 'security-review', role: 'corroborating', matchType: 'hash' }, + ]); + expect(f2?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + ]); + }); + + it('does not attach corroboration to either finding when the same skill ran twice and their ids collide', () => { + const firstExecution = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const secondExecution = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'code-review-strict', + skill: 'code-review', + }); + const corroboratingTrigger = createTrigger({ + id: 'trigger-3', + skillExecutionId: 'exec-3', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const firstResult = createResult({ + triggerId: 'trigger-1', + report: createReport({ skill: 'code-review', findings: [createFinding({ id: 'WRD-001' })] }), + }); + const secondResult = createResult({ + triggerId: 'trigger-2', + report: createReport({ skill: 'code-review', findings: [createFinding({ id: 'WRD-001' })] }), + }); + + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-004' }), + skill: 'security-review', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-001', + existingSkills: ['code-review'], + }, + }, + ]; + + const output = buildFindingsOutputV2( + [firstResult, secondResult], + [firstExecution, secondExecution, corroboratingTrigger], + observations, + { runId: '123' } + ); + + const fromFirstExecution = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-1'); + const fromSecondExecution = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-2'); + + expect(fromFirstExecution?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + ]); + expect(fromSecondExecution?.reportedBy).toEqual([ + { skillExecutionId: 'exec-2', skillName: 'code-review', role: 'primary' }, + ]); + }); + + it('attaches an exact corroboration even when the target skill has multiple executions', () => { + const firstExecution = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const secondExecution = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'code-review-strict', + skill: 'code-review', + }); + const corroboratingTrigger = createTrigger({ + id: 'trigger-3', + skillExecutionId: 'exec-3', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const firstResult = createResult({ + triggerId: 'trigger-1', + report: createReport({ skill: 'code-review', findings: [createFinding({ id: 'WRD-001' })] }), + }); + const secondResult = createResult({ + triggerId: 'trigger-2', + report: createReport({ skill: 'code-review', findings: [createFinding({ id: 'WRD-002' })] }), + }); + + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-004' }), + skill: 'security-review', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-001', + existingSkillExecutionId: 'exec-1', + existingSkills: ['code-review'], + }, + }, + ]; + + const output = buildFindingsOutputV2( + [firstResult, secondResult], + [firstExecution, secondExecution, corroboratingTrigger], + observations, + { runId: '123' } + ); + + const fromFirstExecution = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-1'); + const fromSecondExecution = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-2'); + + expect(fromFirstExecution?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + { skillExecutionId: 'exec-3', skillName: 'security-review', role: 'corroborating', matchType: 'hash' }, + ]); + expect(fromSecondExecution?.reportedBy).toEqual([ + { skillExecutionId: 'exec-2', skillName: 'code-review', role: 'primary' }, + ]); + }); + + it('attaches corroboration when the existing comment has an empty skills array', () => { + const firstTrigger = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const secondTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const result = createResult({ + triggerId: 'trigger-1', + report: createReport({ skill: 'code-review', findings: [createFinding({ id: 'WRD-001' })] }), + }); + + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-002' }), + skill: 'security-review', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-001', + existingSkills: [], + }, + }, + ]; + + const output = buildFindingsOutputV2([result], [firstTrigger, secondTrigger], observations, { runId: '123' }); + + expect(output.findings[0]?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + { skillExecutionId: 'exec-2', skillName: 'security-review', role: 'corroborating', matchType: 'hash' }, + ]); + }); + + it('does not attach corroboration to an unrelated finding sharing an id when the matched comment has an empty skills array', () => { + const firstTrigger = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const secondTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'unrelated-skill-trigger', + skill: 'unrelated-skill', + }); + const thirdTrigger = createTrigger({ + id: 'trigger-3', + skillExecutionId: 'exec-3', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const firstResult = createResult({ + triggerId: 'trigger-1', + report: createReport({ skill: 'code-review', findings: [createFinding({ id: 'WRD-001' })] }), + }); + const secondResult = createResult({ + triggerId: 'trigger-2', + report: createReport({ skill: 'unrelated-skill', findings: [createFinding({ id: 'WRD-001' })] }), + }); + + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-004' }), + skill: 'security-review', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-001', + existingSkillExecutionId: 'exec-1', + existingSkills: [], + }, + }, + ]; + + const output = buildFindingsOutputV2( + [firstResult, secondResult], + [firstTrigger, secondTrigger, thirdTrigger], + observations, + { runId: '123' } + ); + + const fromCodeReview = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-1'); + const fromUnrelated = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-2'); + + expect(fromCodeReview?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + { skillExecutionId: 'exec-3', skillName: 'security-review', role: 'corroborating', matchType: 'hash' }, + ]); + expect(fromUnrelated?.reportedBy).toEqual([ + { skillExecutionId: 'exec-2', skillName: 'unrelated-skill', role: 'primary' }, + ]); + }); + + it('does not cross-attribute two unrelated heuristic matches that coincidentally share a bare id when both existing comments have empty skills arrays', () => { + const firstTrigger = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const secondTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'unrelated-skill-trigger', + skill: 'unrelated-skill', + }); + + const firstResult = createResult({ + triggerId: 'trigger-1', + report: createReport({ skill: 'code-review', findings: [createFinding({ id: 'WRD-001' })] }), + }); + const secondResult = createResult({ + triggerId: 'trigger-2', + report: createReport({ skill: 'unrelated-skill', findings: [createFinding({ id: 'WRD-001' })] }), + }); + + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-001', reportedId: 'WRD-001' }), + skill: 'code-review', + skillExecutionId: 'exec-1', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-001', + existingCommentId: 111, + existingSkills: [], + }, + }, + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-001', reportedId: 'WRD-001' }), + skill: 'unrelated-skill', + skillExecutionId: 'exec-2', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-001', + existingCommentId: 222, + existingSkills: [], + }, + }, + ]; + + const output = buildFindingsOutputV2( + [firstResult, secondResult], + [firstTrigger, secondTrigger], + observations, + { runId: '123' } + ); + + const fromCodeReview = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-1'); + const fromUnrelated = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-2'); + + expect(fromCodeReview?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + ]); + expect(fromUnrelated?.reportedBy).toEqual([ + { skillExecutionId: 'exec-2', skillName: 'unrelated-skill', role: 'primary' }, + ]); + }); + + it('claims a heuristic corroborator for at most one target when two unrelated fresh findings share a bare id and neither has an anchor', () => { + // Unlike the two tests above, neither exec-1 nor exec-2 dedupes against + // anything itself this run - both findings are fresh, so neither has an + // ownAnchor to narrow the security-review match against. Combined with + // an empty existingSkills (permissive by design), nothing but claiming + // stops the same corroborator from attaching to both. + const firstTrigger = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const secondTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'perf-review-trigger', + skill: 'perf-review', + }); + const thirdTrigger = createTrigger({ + id: 'trigger-3', + skillExecutionId: 'exec-3', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const firstResult = createResult({ + triggerId: 'trigger-1', + report: createReport({ skill: 'code-review', findings: [createFinding({ id: 'WRD-001' })] }), + }); + const secondResult = createResult({ + triggerId: 'trigger-2', + report: createReport({ skill: 'perf-review', findings: [createFinding({ id: 'WRD-001' })] }), + }); + + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-005' }), + skill: 'security-review', + skillExecutionId: 'exec-3', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-001', + existingSkills: [], + }, + }, + ]; + + const output = buildFindingsOutputV2( + [firstResult, secondResult], + [firstTrigger, secondTrigger, thirdTrigger], + observations, + { runId: '123' } + ); + + const fromCodeReview = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-1'); + const fromPerfReview = output.findings.find((f) => f.provenance.originSkillExecutionId === 'exec-2'); + + const claimedByCodeReview = fromCodeReview?.reportedBy.some((r) => r.skillExecutionId === 'exec-3') ?? false; + const claimedByPerfReview = fromPerfReview?.reportedBy.some((r) => r.skillExecutionId === 'exec-3') ?? false; + + expect([claimedByCodeReview, claimedByPerfReview].filter(Boolean)).toHaveLength(1); + }); + + it('does not list a finding as its own corroborator when it dedupes against its own prior posting', () => { + const trigger = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const finding = createFinding({ id: 'WRD-001', reportedId: 'EXISTING-001' }); + + const result = createResult({ + triggerId: 'trigger-1', + report: createReport({ skill: 'code-review', findings: [finding] }), + }); + + // Single-run mode: this finding's own continuity dedupe (matching its own + // prior posting) is recorded as a 'deduped' observation whose + // existingFindingId is the finding's own reportedId. + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding, + skill: 'code-review', + skillExecutionId: 'exec-1', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'EXISTING-001', + existingSkills: ['code-review'], + }, + }, + ]; + + const output = buildFindingsOutputV2([result], [trigger], observations, { runId: '123' }); + + expect(output.findings[0]?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + ]); + }); + + it('records verifier rejections in discardedFindings', () => { + const trigger = createTrigger(); + const result = createResult({ + findingProcessingEvents: [ + { + stage: 'verification', + action: 'rejected', + finding: createFinding({ id: 'WRD-003' }), + reason: 'mitigated upstream', + model: 'claude-haiku-4-5', + }, + ], + }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + expect(output.discardedFindings).toEqual([ + { + originSkillExecutionId: 'exec-1', + stage: 'verification_rejected', + severity: 'high', + title: 'Finding title', + location: { path: 'src/index.ts', startLine: 1 }, + model: 'claude-haiku-4-5', + reason: 'mitigated upstream', + }, + ]); + }); + + it('adds corroborating attribution when another skill matches an existing finding', () => { + const primaryTrigger = createTrigger(); + const corroboratingTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-004' }), + skill: 'security-review', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'WRD-001', + }, + }, + ]; + + const output = buildFindingsOutputV2( + [createResult()], + [primaryTrigger, corroboratingTrigger], + observations, + { runId: '123' } + ); + + expect(output.findings[0]?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + { skillExecutionId: 'exec-2', skillName: 'security-review', role: 'corroborating', matchType: 'hash' }, + ]); + expect(output.summary.byOutcome.deduped).toBe(1); + }); + + it('collapses repeat corroboration from the same skill execution into one reportedBy entry', () => { + const primaryTrigger = createTrigger(); + const corroboratingTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'security-review-trigger', + skill: 'security-review', + }); + + const observations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-004' }), + skill: 'security-review', + dedupe: { source: 'warden', matchType: 'hash', existingFindingId: 'WRD-001' }, + }, + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-005' }), + skill: 'security-review', + dedupe: { source: 'warden', matchType: 'hash', existingFindingId: 'WRD-001' }, + }, + ]; + + const output = buildFindingsOutputV2( + [createResult()], + [primaryTrigger, corroboratingTrigger], + observations, + { runId: '123' } + ); + + expect(output.findings[0]?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + { skillExecutionId: 'exec-2', skillName: 'security-review', role: 'corroborating', matchType: 'hash' }, + ]); + }); + + it('attributes an observation to its own execution when two triggers share a skill name', () => { + const firstExecution = createTrigger({ id: 'trigger-1', skillExecutionId: 'exec-1', skill: 'code-review' }); + const secondExecution = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'code-review-strict', + skill: 'code-review', + }); + + const observations: FindingObservation[] = [ + { + outcome: 'skipped', + finding: createFinding({ id: 'WRD-501' }), + skill: 'code-review', + skillExecutionId: 'exec-2', + skippedReason: 'max_findings', + }, + ]; + + const output = buildFindingsOutputV2( + [createResult()], + [firstExecution, secondExecution], + observations, + { runId: '123' } + ); + + expect(output.findingObservations[0]?.origin).toEqual({ + skillExecutionId: 'exec-2', + skillName: 'code-review', + }); + }); +}); + +describe('fromAuxiliaryUsageEntries', () => { + it('round-trips through buildFindingsOutputV2 back into the record-keyed shape SkillReport expects', () => { + const trigger = createTrigger(); + const result = createResult({ + report: createReport({ + auxiliaryUsage: { dedup: { inputTokens: 10, outputTokens: 5, costUSD: 0.002 } }, + auxiliaryUsageAttribution: { dedup: { model: 'claude-haiku-4-5', runtime: 'pi' } }, + }), + }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + const { usage, attribution } = fromAuxiliaryUsageEntries(output.skillExecutions[0]?.auxiliaryUsage); + + expect(usage).toEqual({ dedup: { inputTokens: 10, outputTokens: 5, costUSD: 0.002 } }); + expect(attribution).toEqual({ dedup: { model: 'claude-haiku-4-5', runtime: 'pi' } }); + }); + + it('returns undefined for both when there are no entries', () => { + expect(fromAuxiliaryUsageEntries(undefined)).toEqual({ usage: undefined, attribution: undefined }); + expect(fromAuxiliaryUsageEntries([])).toEqual({ usage: undefined, attribution: undefined }); + }); +}); + +describe('patchFindingsOutputV2Observations', () => { + it('is a no-op when patched with no report-phase observations (build/patch parity)', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-002', severity: 'medium' }); + const result = createResult({ + report: createReport({ findings: [finding] }), + findingProcessingEvents: [ + { + stage: 'verification', + action: 'revised', + finding: createFinding({ id: 'WRD-002', severity: 'high', title: 'Original title' }), + replacement: finding, + reason: 'narrower scope', + model: 'claude-haiku-4-5', + }, + ], + }); + + const analyzePhaseOutput = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + const patched = patchFindingsOutputV2Observations(analyzePhaseOutput, [result], [trigger], []); + + expect(patched).toEqual(analyzePhaseOutput); + }); + + it('backfills skillExecutions usage/auxiliaryUsage from live report-phase posting costs', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-401' }); + + const analyzePhaseOutput = buildFindingsOutputV2( + [createResult({ report: createReport({ findings: [finding] }) })], + [trigger], + [], + { runId: '123' } + ); + expect(analyzePhaseOutput.skillExecutions[0]?.auxiliaryUsage).toBeUndefined(); + + // Posting (dedupe/consolidate) merges auxiliary usage onto the live report + // after the analyze-phase payload above was already built (see poster.ts). + const postedResult = createResult({ + report: createReport({ + findings: [finding], + auxiliaryUsage: { dedupe: { inputTokens: 100, outputTokens: 20, costUSD: 0.01 } }, + auxiliaryUsageAttribution: { dedupe: { model: 'claude-haiku-4-5' } }, + }), + }); + + const patched = patchFindingsOutputV2Observations(analyzePhaseOutput, [postedResult], [trigger], []); + + expect(patched.skillExecutions[0]?.auxiliaryUsage).toEqual([ + { + agent: 'dedupe', + model: 'claude-haiku-4-5', + usage: { inputTokens: 100, outputTokens: 20, costUSD: 0.01 }, + }, + ]); + }); + + it('backfills skillExecutions checkRunUrl/checkRunId from report-phase check creation', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-402' }); + + const analyzePhaseOutput = buildFindingsOutputV2( + [createResult({ report: createReport({ findings: [finding] }) })], + [trigger], + [], + { runId: '123' } + ); + expect(analyzePhaseOutput.skillExecutions[0]?.checkRunUrl).toBeUndefined(); + expect(analyzePhaseOutput.skillExecutions[0]?.checkRunId).toBeUndefined(); + + // Report mode creates its skill checks as already-completed check runs + // only after the analyze-phase payload above was already built (see + // createCompletedSkillChecksForReport in pr-workflow.ts). + const postedResult = createResult({ + report: createReport({ findings: [finding] }), + checkRunUrl: 'https://github.com/getsentry/warden/runs/555', + checkRunId: 555, + }); + + const patched = patchFindingsOutputV2Observations(analyzePhaseOutput, [postedResult], [trigger], []); + + expect(patched.skillExecutions[0]?.checkRunUrl).toBe('https://github.com/getsentry/warden/runs/555'); + expect(patched.skillExecutions[0]?.checkRunId).toBe(555); + }); + + it('backfills skillExecutions reviewEvent/checkConclusion from the report-phase posting outcome', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-403', severity: 'high', confidence: 'low' }); + + const analyzePhaseOutput = buildFindingsOutputV2( + [createResult({ + report: createReport({ findings: [finding] }), + failOn: 'high', + failCheck: true, + renderResult: { review: { event: 'REQUEST_CHANGES', body: '', comments: [] }, summaryComment: '' }, + })], + [trigger], + [], + { runId: '123' } + ); + expect(analyzePhaseOutput.skillExecutions[0]?.reviewEvent).toBe('REQUEST_CHANGES'); + expect(analyzePhaseOutput.skillExecutions[0]?.checkConclusion).toBe('failure'); + + // Report-phase dedup/consolidate can shrink the posted review (or a + // trigger's own minConfidence can differ once replayed), so the real + // posting outcome must win over the analyze-phase payload's assumption. + const postedResult = createResult({ + report: createReport({ findings: [finding] }), + failOn: 'high', + failCheck: true, + minConfidence: 'high', + renderResult: { review: { event: 'COMMENT', body: '', comments: [] }, summaryComment: '' }, + }); + + const patched = patchFindingsOutputV2Observations(analyzePhaseOutput, [postedResult], [trigger], []); + + expect(patched.skillExecutions[0]?.reviewEvent).toBe('COMMENT'); + expect(patched.skillExecutions[0]?.checkConclusion).toBe('success'); + }); + + it('backfills a finding githubCommentId/githubCommentUrl from report-phase posting', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-502' }); + + const analyzePhaseOutput = buildFindingsOutputV2( + [createResult({ report: createReport({ findings: [finding] }) })], + [trigger], + [], + { runId: '123' } + ); + expect(analyzePhaseOutput.findings[0]?.githubCommentId).toBeUndefined(); + + const reportPhaseObservations: FindingObservation[] = [ + { + outcome: 'posted', + finding, + skill: 'code-review', + skillExecutionId: 'exec-1', + githubCommentId: 77, + githubCommentUrl: 'https://github.com/getsentry/warden/pull/1#discussion_r77', + }, + ]; + const patched = patchFindingsOutputV2Observations(analyzePhaseOutput, [createResult()], [trigger], reportPhaseObservations); + + expect(patched.findings[0]?.githubCommentId).toBe(77); + expect(patched.findings[0]?.githubCommentUrl).toBe('https://github.com/getsentry/warden/pull/1#discussion_r77'); + }); + + it('preserves skillExecutions/findings/discardedFindings/provenance while updating only observations and byOutcome', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-002', severity: 'medium' }); + const result = createResult({ + report: createReport({ findings: [finding] }), + findingProcessingEvents: [ + { + stage: 'verification', + action: 'revised', + finding: createFinding({ id: 'WRD-002', severity: 'high', title: 'Original title' }), + replacement: finding, + reason: 'narrower scope', + model: 'claude-haiku-4-5', + }, + { + stage: 'verification', + action: 'rejected', + finding: createFinding({ id: 'WRD-003' }), + reason: 'mitigated upstream', + model: 'claude-haiku-4-5', + }, + ], + }); + + const analyzePhaseOutput = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + const reportPhaseObservations: FindingObservation[] = [ + { outcome: 'posted', finding, skill: 'code-review' }, + ]; + const patched = patchFindingsOutputV2Observations(analyzePhaseOutput, [result], [trigger], reportPhaseObservations); + + // The parts that can only be reconstructed from findingProcessingEvents + // (unavailable during report-mode replay) must survive unchanged. + expect(patched.skillExecutions).toEqual(analyzePhaseOutput.skillExecutions); + expect(patched.findings).toEqual(analyzePhaseOutput.findings); + expect(patched.discardedFindings).toEqual(analyzePhaseOutput.discardedFindings); + expect(patched.findings[0]?.provenance.verification?.outcome).toBe('revised'); + + // Only the observation-derived parts reflect the new (report-phase) data. + expect(patched.findingObservations).toEqual([ + expect.objectContaining({ outcome: 'posted', finding: expect.objectContaining({ id: 'WRD-002' }) }), + ]); + expect(patched.summary.byOutcome).toEqual({ posted: 1, deduped: 0, skipped: 0, resolved: 0, failed: 0 }); + expect(patched.summary.totalFindings).toBe(analyzePhaseOutput.summary.totalFindings); + }); + + it('adds corroborating attribution discovered at report/post time without touching provenance', () => { + const primaryTrigger = createTrigger(); + const corroboratingTrigger = createTrigger({ + id: 'trigger-2', + skillExecutionId: 'exec-2', + name: 'security-review-trigger', + skill: 'security-review', + }); + const finding = createFinding({ id: 'WRD-101' }); + + const analyzePhaseOutput = buildFindingsOutputV2( + [createResult({ report: createReport({ findings: [finding] }) })], + [primaryTrigger], + [], + { runId: '123' } + ); + expect(analyzePhaseOutput.findings[0]?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + ]); + + // Cross-skill dedup only happens during posting, so this observation only + // becomes known during the report phase, after analyze-phase output exists. + const reportPhaseObservations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: createFinding({ id: 'WRD-201' }), + skill: 'security-review', + dedupe: { source: 'warden', matchType: 'semantic', existingFindingId: 'WRD-101' }, + }, + ]; + + const patched = patchFindingsOutputV2Observations( + analyzePhaseOutput, [], [primaryTrigger, corroboratingTrigger], reportPhaseObservations + ); + + expect(patched.findings[0]?.reportedBy).toEqual([ + { skillExecutionId: 'exec-1', skillName: 'code-review', role: 'primary' }, + { skillExecutionId: 'exec-2', skillName: 'security-review', role: 'corroborating', matchType: 'semantic' }, + ]); + expect(patched.findings[0]?.provenance).toEqual(analyzePhaseOutput.findings[0]?.provenance); + }); + + it('adds reportedId without ever renaming findings[].id or skillExecutions[].findingIds', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-301' }); + + const analyzePhaseOutput = buildFindingsOutputV2( + [createResult({ report: createReport({ findings: [finding] }) })], + [trigger], + [], + { runId: '123' } + ); + expect(analyzePhaseOutput.findings[0]?.id).toBe('WRD-301'); + expect(analyzePhaseOutput.skillExecutions[0]?.findingIds).toEqual(['WRD-301']); + + // Report-time dedupe against an existing GitHub comment sets `reportedId` + // on the finding (the analyze-phase payload being patched here still has + // no reportedId, since dedupe hadn't happened yet) - `id` itself is + // stable and never needs renaming. + const reportPhaseObservations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: { ...finding, reportedId: 'WRZ-XPL' }, + skill: 'code-review', + dedupe: { source: 'warden', matchType: 'hash', existingFindingId: 'WRZ-XPL' }, + }, + ]; + + const patched = patchFindingsOutputV2Observations(analyzePhaseOutput, [], [trigger], reportPhaseObservations); + + expect(patched.findings[0]?.id).toBe('WRD-301'); + expect(patched.findings[0]?.reportedId).toBe('WRZ-XPL'); + expect(patched.skillExecutions[0]?.findingIds).toEqual(['WRD-301']); + }); + + it('never lets a report-time dedupe in one skill rename an unrelated finding that shares its id in another skill', () => { + const triggerA = createTrigger({ id: 'trigger-a', skillExecutionId: 'exec-a', skill: 'skill-a' }); + const triggerB = createTrigger({ id: 'trigger-b', skillExecutionId: 'exec-b', skill: 'skill-b' }); + const findingA = createFinding({ id: 'SHARED-ID' }); + const findingB = createFinding({ id: 'SHARED-ID' }); + + const analyzePhaseOutput = buildFindingsOutputV2( + [ + createResult({ report: createReport({ skill: 'skill-a', findings: [findingA] }), triggerId: 'trigger-a', skillExecutionId: 'exec-a' }), + createResult({ report: createReport({ skill: 'skill-b', findings: [findingB] }), triggerId: 'trigger-b', skillExecutionId: 'exec-b' }), + ], + [triggerA, triggerB], + [], + { runId: '123' } + ); + expect(analyzePhaseOutput.findings).toHaveLength(2); + expect(analyzePhaseOutput.findings.every((f) => f.id === 'SHARED-ID')).toBe(true); + + // Only skill A's finding gets deduped this report step. + const reportPhaseObservations: FindingObservation[] = [ + { + outcome: 'deduped', + finding: { ...findingA, reportedId: 'WRZ-XPL' }, + skill: 'skill-a', + skillExecutionId: 'exec-a', + dedupe: { source: 'warden', matchType: 'hash', existingFindingId: 'WRZ-XPL' }, + }, + ]; + + const patched = patchFindingsOutputV2Observations(analyzePhaseOutput, [], [triggerA, triggerB], reportPhaseObservations); + + const patchedA = patched.findings.find((f) => f.reportedBy.some((r) => r.skillName === 'skill-a')); + const patchedB = patched.findings.find((f) => f.reportedBy.some((r) => r.skillName === 'skill-b')); + expect(patchedA?.reportedId).toBe('WRZ-XPL'); + expect(patchedB?.reportedId).toBeUndefined(); + }); +}); + +describe('buildFindingsOutputV2 kept verification provenance', () => { + it('records a kept verdict in provenance.verification', () => { + const trigger = createTrigger(); + const finding = createFinding({ id: 'WRD-401' }); + const result = createResult({ + report: createReport({ findings: [finding] }), + findingProcessingEvents: [ + { + stage: 'verification', + action: 'kept', + finding, + reason: 'still real after tracing', + model: 'claude-haiku-4-5', + }, + ], + }); + + const output = buildFindingsOutputV2([result], [trigger], [], { runId: '123' }); + + expect(output.findings[0]?.provenance.verification).toEqual({ + outcome: 'kept', + model: 'claude-haiku-4-5', + runtime: undefined, + }); + }); +}); diff --git a/packages/warden/src/action/reporting/output-v2.ts b/packages/warden/src/action/reporting/output-v2.ts new file mode 100644 index 000000000..a9f2d89b4 --- /dev/null +++ b/packages/warden/src/action/reporting/output-v2.ts @@ -0,0 +1,1232 @@ +import { z } from 'zod'; +import { + ConfidenceSchema, + ConfidenceThresholdSchema, + filterFindings, + GitHubEventTypeSchema, + LocationSchema, + SeveritySchema, + SeverityThresholdSchema, + SkillErrorSchema, + SourceSnippetSchema, + UsageStatsSchema, + VerifierRejectionsSchema, +} from '../../types/index.js'; +import type { + AuxiliaryUsageAttributionMap, + AuxiliaryUsageMap, + EventContext, + Finding, + Severity, + SeverityThreshold, + SkillReport, +} from '../../types/index.js'; +import type { ResolvedTrigger } from '../../config/loader.js'; +import { matchPullRequestState } from '../../triggers/matcher.js'; +import type { TriggerResult } from '../triggers/executor.js'; +import { buildConfiguredSkillsList, serializeTriggerError } from './output.js'; +import { generateContentHash } from '../../output/dedup.js'; +import { determineConclusion } from '../../output/github-checks.js'; +import { getVersion } from '../../utils/version.js'; +import { displayFindingId } from '../../cli/output/formatters.js'; +import type { FindingObservation } from './outcomes.js'; + +export const SeverityBreakdownSchema = z.object({ + high: z.number().int().nonnegative(), + medium: z.number().int().nonnegative(), + low: z.number().int().nonnegative(), +}); +export type SeverityBreakdown = z.infer; + +const HarnessSchema = z.object({ + name: z.literal('warden'), + version: z.string(), + actionRef: z.string().optional(), +}); + +const RepositorySchema = z.object({ + owner: z.string(), + name: z.string(), + fullName: z.string(), +}); + +const PullRequestEnvelopeSchema = z.object({ + number: z.number().int(), + author: z.string(), + title: z.string(), + baseBranch: z.string(), + headBranch: z.string(), + headSha: z.string(), +}); + +const ConfiguredSkillSchema = z.object({ + name: z.string(), + triggered: z.boolean(), +}); + +export const SkippedTriggerReasonSchema = z.enum([ + 'no_event_match', + 'path_filter', + 'draft_state', + 'label_mismatch', + 'no_changes', + 'pending', +]); + +const SkippedTriggerSchema = z.object({ + skillName: z.string(), + triggerId: z.string().optional(), + triggerName: z.string().optional(), + reason: SkippedTriggerReasonSchema, +}); + +const TriggerErrorSchema = z.object({ + name: z.string().optional(), + message: z.string(), +}); + +export const TriggerRunResultV2Schema = z.discriminatedUnion('status', [ + z.object({ + status: z.literal('success'), + triggerId: z.string().optional(), + triggerName: z.string(), + skillName: z.string(), + }), + z.object({ + status: z.literal('error'), + triggerId: z.string().optional(), + triggerName: z.string(), + skillName: z.string(), + error: TriggerErrorSchema, + }), +]); + +const ResolvedDefaultsSchema = z.object({ + failOn: SeverityThresholdSchema.optional(), + reportOn: SeverityThresholdSchema.optional(), + minConfidence: ConfidenceThresholdSchema.optional(), + model: z.string().optional(), + auxiliaryModel: z.string().optional(), + synthesisModel: z.string().optional(), + runtime: z.string().optional(), + verifyFindings: z.boolean().optional(), + failCheck: z.boolean().optional(), + requestChanges: z.boolean().optional(), + maxFindings: z.number().int().nonnegative().optional(), +}); + +export const WardenMetadataSchema = z.object({ + schemaVersion: z.literal('2'), + runId: z.string(), + runAttempt: z.string().optional(), + generatedAt: z.string().datetime(), + harness: HarnessSchema, + repository: RepositorySchema, + event: GitHubEventTypeSchema, + pullRequest: PullRequestEnvelopeSchema.optional(), + configuredSkills: z.array(ConfiguredSkillSchema).optional(), + skippedTriggers: z.array(SkippedTriggerSchema).optional(), + triggerResults: z.array(TriggerRunResultV2Schema).optional(), + resolvedDefaults: ResolvedDefaultsSchema.optional(), +}); +export type WardenMetadata = z.infer; + +const AuxiliaryUsageEntrySchema = z.object({ + agent: z.string(), + model: z.string().optional(), + runtime: z.string().optional(), + usage: UsageStatsSchema, +}); + +export const SkillExecutionSchema = z.object({ + skillExecutionId: z.string(), + skillName: z.string(), + triggerId: z.string().optional(), + triggerName: z.string().optional(), + model: z.string().optional(), + models: z.array(z.string()).optional(), + runtime: z.string().optional(), + auxiliaryModel: z.string().optional(), + synthesisModel: z.string().optional(), + summary: z.string(), + durationMs: z.number().nonnegative().optional(), + usage: UsageStatsSchema.optional(), + auxiliaryUsage: z.array(AuxiliaryUsageEntrySchema).optional(), + findingsBySeverity: SeverityBreakdownSchema, + findingIds: z.array(z.string()), + failedHunks: z.number().int().nonnegative().optional(), + failedExtractions: z.number().int().nonnegative().optional(), + error: SkillErrorSchema.optional(), + verifierRejections: VerifierRejectionsSchema.optional(), + checkRunUrl: z.string().optional(), + checkRunId: z.number().int().positive().optional(), + reviewEvent: z.enum(['APPROVE', 'REQUEST_CHANGES', 'COMMENT']).optional(), + checkConclusion: z.enum(['success', 'failure', 'neutral', 'cancelled']).optional(), + issueNumber: z.number().int().positive().optional(), + issueUrl: z.string().optional(), +}); +export type SkillExecution = z.infer; + +const FindingSnapshotSchema = z.object({ + title: z.string(), + description: z.string(), + severity: SeveritySchema, + confidence: ConfidenceSchema.optional(), +}); + +const VerificationStageSchema = z.discriminatedUnion('outcome', [ + z.object({ + outcome: z.literal('kept'), + model: z.string().optional(), + runtime: z.string().optional(), + }), + z.object({ + outcome: z.literal('revised'), + model: z.string().optional(), + runtime: z.string().optional(), + evidence: z.string().optional(), + before: FindingSnapshotSchema, + }), +]); + +const MergeStageSchema = z.object({ + model: z.string().optional(), + runtime: z.string().optional(), + absorbedFindingIds: z.array(z.string()), +}); + +export type VerificationStage = z.infer; +export type MergeStage = z.infer; + +const FindingProvenanceSchema = z.object({ + originSkillExecutionId: z.string(), + originModel: z.string().optional(), + verification: VerificationStageSchema.optional(), + merge: MergeStageSchema.optional(), +}); +export type FindingProvenance = z.infer; + +const FindingAttributionSchema = z.object({ + skillExecutionId: z.string(), + skillName: z.string(), + role: z.enum(['primary', 'corroborating']), + matchType: z.enum(['hash', 'semantic']).optional(), +}); +export type FindingAttribution = z.infer; + +export const ExportedFindingV2Schema = z.object({ + id: z.string(), + reportedId: z.string().optional(), + contentHash: z.string(), + severity: SeveritySchema, + confidence: ConfidenceSchema.optional(), + title: z.string(), + description: z.string(), + verification: z.string().optional(), + location: LocationSchema.optional(), + additionalLocations: z.array(LocationSchema).optional(), + sourceSnippet: SourceSnippetSchema.optional(), + reportedBy: z.array(FindingAttributionSchema).min(1), + provenance: FindingProvenanceSchema, + githubCommentId: z.number().int().positive().optional(), + githubCommentUrl: z.string().optional(), +}); +export type ExportedFindingV2 = z.infer; + +export const DiscardedFindingSchema = z.object({ + originSkillExecutionId: z.string(), + stage: z.enum(['verification_rejected', 'merge_absorbed']), + severity: SeveritySchema, + title: z.string(), + location: LocationSchema.optional(), + model: z.string().optional(), + reason: z.string().optional(), + survivorFindingId: z.string().optional(), +}); +export type DiscardedFinding = z.infer; + +const FindingOriginSchema = z.object({ + skillExecutionId: z.string(), + skillName: z.string(), +}); + +export const DedupeDetailV2Schema = z.object({ + source: z.enum(['warden', 'external']), + matchType: z.enum(['hash', 'semantic']), + existingFindingId: z.string().optional(), + existingSkillExecutionId: z.string().optional(), + existingCommentId: z.number().int().positive().optional(), + existingThreadId: z.string().optional(), + existingResolved: z.boolean().optional(), + existingSkills: z.array(z.string()).optional(), + actor: z.string().optional(), +}); +export type DedupeDetailV2 = z.infer; + +const ObservedFindingSchema = z.object({ + id: z.string(), + reportedId: z.string().optional(), + severity: SeveritySchema, + confidence: ConfidenceSchema.optional(), + title: z.string(), + description: z.string(), + location: LocationSchema.optional(), + elapsedMs: z.number().nonnegative().optional(), +}); + +export const FindingObservationV2Schema = z.discriminatedUnion('outcome', [ + z.object({ + outcome: z.literal('posted'), + origin: FindingOriginSchema, + finding: ObservedFindingSchema, + githubCommentId: z.number().int().positive().optional(), + githubCommentUrl: z.string().optional(), + }), + z.object({ + outcome: z.literal('deduped'), + origin: FindingOriginSchema, + finding: ObservedFindingSchema, + dedupe: DedupeDetailV2Schema, + }), + z.object({ + outcome: z.literal('skipped'), + origin: FindingOriginSchema, + finding: ObservedFindingSchema, + skippedReason: z.enum(['max_findings', 'duplicate_in_batch', 'no_inline_location', 'review_not_posted']), + }), + z.object({ + outcome: z.literal('resolved'), + origin: FindingOriginSchema, + finding: ObservedFindingSchema, + resolvedReason: z.enum(['fix_evaluation', 'stale_check']), + }), + z.object({ + outcome: z.literal('failed'), + origin: FindingOriginSchema, + finding: ObservedFindingSchema, + }), +]); +export type FindingObservationV2 = z.infer; + +const SummarySchema = z.object({ + totalFindings: z.number().int().nonnegative(), + totalSkillExecutions: z.number().int().nonnegative(), + bySeverity: SeverityBreakdownSchema, + byOutcome: z.object({ + posted: z.number().int().nonnegative(), + deduped: z.number().int().nonnegative(), + skipped: z.number().int().nonnegative(), + resolved: z.number().int().nonnegative(), + failed: z.number().int().nonnegative(), + }), +}); +export type SummaryV2 = z.infer; + +export const WardenFindingsSchemaV2 = z.object({ + schemaVersion: z.literal('2'), + runId: z.string(), + skillExecutions: z.array(SkillExecutionSchema), + findings: z.array(ExportedFindingV2Schema), + discardedFindings: z.array(DiscardedFindingSchema).optional(), + findingObservations: z.array(FindingObservationV2Schema), + summary: SummarySchema, +}); +export type WardenFindingsV2 = z.infer; + +// ----------------------------------------------------------------------------- +// Builders +// ----------------------------------------------------------------------------- + +function severityBreakdown(items: { severity: Severity }[]): SeverityBreakdown { + return { + high: items.filter((i) => i.severity === 'high').length, + medium: items.filter((i) => i.severity === 'medium').length, + low: items.filter((i) => i.severity === 'low').length, + }; +} + +function toAuxiliaryUsageEntries( + usage: AuxiliaryUsageMap | undefined, + attribution: AuxiliaryUsageAttributionMap | undefined +): z.infer[] { + if (!usage) return []; + return Object.entries(usage).map(([agent, agentUsage]) => { + const agentAttribution = attribution?.[agent]; + return { + agent, + model: agentAttribution?.model ?? agentAttribution?.models?.[0], + runtime: agentAttribution?.runtime ?? agentAttribution?.runtimes?.[0], + usage: agentUsage, + }; + }); +} + +/** Inverse of {@link toAuxiliaryUsageEntries} — rebuilds the record-keyed shape SkillReport expects. */ +export function fromAuxiliaryUsageEntries( + entries: z.infer[] | undefined +): { usage: AuxiliaryUsageMap | undefined; attribution: AuxiliaryUsageAttributionMap | undefined } { + if (!entries || entries.length === 0) return { usage: undefined, attribution: undefined }; + + const usage: AuxiliaryUsageMap = {}; + const attribution: AuxiliaryUsageAttributionMap = {}; + for (const entry of entries) { + usage[entry.agent] = entry.usage; + if (entry.model || entry.runtime) { + attribution[entry.agent] = { model: entry.model, runtime: entry.runtime }; + } + } + return { usage, attribution: Object.keys(attribution).length > 0 ? attribution : undefined }; +} + +function deriveSkippedReason( + trigger: ResolvedTrigger, + context: EventContext +): z.infer { + if (trigger.type === 'local') return 'no_event_match'; + if (trigger.type === 'schedule') { + return context.eventType === 'schedule' ? 'no_changes' : 'no_event_match'; + } + // schedule.ts only ever evaluates type: 'schedule' triggers - a wildcard + // trigger never reaches matchTrigger's path-filter check in a scheduled + // run, so it isn't excluded by paths, it's excluded by event type entirely + // (same as the 'local'/'pull_request' branches above for a schedule event). + if (trigger.type === '*' && context.eventType === 'schedule') return 'no_event_match'; + if (trigger.type === 'pull_request') { + if (context.eventType !== 'pull_request') return 'no_event_match'; + if (!trigger.actions?.includes(context.action)) return 'no_event_match'; + if (!matchPullRequestState(trigger, context)) { + if (context.action === 'labeled' && trigger.labels !== undefined) { + const eventLabelMatches = context.label !== undefined && trigger.labels.includes(context.label); + if (!eventLabelMatches) return 'label_mismatch'; + } + const labels = context.pullRequest?.labels ?? []; + const labelMatches = trigger.labels?.some((label) => labels.includes(label)); + if (trigger.labels !== undefined && !labelMatches) return 'label_mismatch'; + return 'draft_state'; + } + } + return 'path_filter'; +} + +export interface BuildMetadataOutputV2Options { + runId: string; + runAttempt?: string; + generatedAt?: string; + actionRef?: string; + /** Action-level fallback used by every trigger via `trigger.failOn ?? inputs.failOn`. */ + failOn?: SeverityThreshold; + /** Action-level fallback used by every trigger via `trigger.reportOn ?? inputs.reportOn`. */ + reportOn?: SeverityThreshold; + /** Action-level fallback used by every trigger via `trigger.failCheck ?? inputs.failCheck`. */ + failCheck?: boolean; + /** Action-level fallback used by every trigger via `trigger.requestChanges ?? inputs.requestChanges`. */ + requestChanges?: boolean; + /** Action-level fallback used by every trigger via `trigger.maxFindings ?? inputs.maxFindings`. */ + maxFindings?: number; + /** Triggers not yet attempted this run (schedule.ts's sequential loop hasn't reached them), reported as 'pending' instead of a guessed skip reason. */ + pendingTriggerIds?: Set; +} + +/** Build the schema-v2 metadata output: static run/repo/harness identity plus the resolved trigger roster. */ +export function buildMetadataOutputV2( + context: EventContext, + resolvedTriggers: ResolvedTrigger[], + matchedTriggers: ResolvedTrigger[], + results: TriggerResult[], + options: BuildMetadataOutputV2Options +): WardenMetadata { + const matchedIds = new Set(matchedTriggers.map((t) => t.id)); + const pendingIds = options.pendingTriggerIds; + const skippedTriggers = resolvedTriggers + .filter((t) => !matchedIds.has(t.id)) + .map((t) => ({ + skillName: t.skill, + triggerId: t.id, + triggerName: t.name, + reason: pendingIds?.has(t.id) ? ('pending' as const) : deriveSkippedReason(t, context), + })); + + const triggerResults = results.map((r) => + r.error + ? { + status: 'error' as const, + triggerId: r.triggerId, + triggerName: r.triggerName, + skillName: r.skillName, + error: serializeTriggerError(r.error), + } + : { + status: 'success' as const, + triggerId: r.triggerId, + triggerName: r.triggerName, + skillName: r.skillName, + } + ); + + const primary = matchedTriggers[0]; + + return WardenMetadataSchema.parse({ + schemaVersion: '2', + runId: options.runId, + runAttempt: options.runAttempt, + generatedAt: options.generatedAt ?? new Date().toISOString(), + harness: { + name: 'warden', + version: getVersion(), + actionRef: options.actionRef, + }, + repository: { + owner: context.repository.owner, + name: context.repository.name, + fullName: context.repository.fullName, + }, + event: context.eventType, + ...(context.pullRequest && { + pullRequest: { + number: context.pullRequest.number, + author: context.pullRequest.author, + title: context.pullRequest.title, + baseBranch: context.pullRequest.baseBranch, + headBranch: context.pullRequest.headBranch, + headSha: context.pullRequest.headSha, + }, + }), + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + skippedTriggers, + triggerResults, + ...(primary && { + resolvedDefaults: { + failOn: primary.failOn ?? options.failOn, + reportOn: primary.reportOn ?? options.reportOn, + minConfidence: primary.minConfidence ?? 'medium', + model: primary.model, + auxiliaryModel: primary.auxiliaryModel, + synthesisModel: primary.synthesisModel, + runtime: primary.runtime, + verifyFindings: primary.verifyFindings, + failCheck: primary.failCheck ?? options.failCheck, + requestChanges: primary.requestChanges ?? options.requestChanges, + maxFindings: primary.maxFindings ?? options.maxFindings, + }, + }), + }); +} + +/** skillExecutionId per skill name, restricted to names with exactly one current execution. */ +export function skillExecutionIdByNameFrom(matchedTriggers: ResolvedTrigger[]): Map { + const counts = new Map(); + const idByName = new Map(); + for (const t of matchedTriggers) { + counts.set(t.skill, (counts.get(t.skill) ?? 0) + 1); + if (!idByName.has(t.skill)) { + idByName.set(t.skill, t.skillExecutionId); + } + } + + const skillExecutionIdByName = new Map(); + for (const [name, id] of idByName) { + if (counts.get(name) === 1) skillExecutionIdByName.set(name, id); + } + return skillExecutionIdByName; +} + +/** + * Resolves an observation's own execution id: its own id, then a by-name + * fallback. Deliberately falls back to '' (never the bare skill name) when + * neither is known - an observation can describe a comment from a trigger + * no longer in the current config, so guessing a truthy-but-wrong id here + * risks a false match at a `${skillExecutionId}:${id}` compound key. + */ +function resolveObservationSkillExecutionId( + ownId: string | undefined, + skillName: string | undefined, + skillExecutionIdByName: Map +): string { + return ownId ?? skillExecutionIdByName.get(skillName ?? '') ?? ''; +} + +/** + * Resolves a current result's own execution id: its own id, then a by-name + * fallback, then the skill name itself. Unlike the observation-side + * resolver, this always describes a result that just ran in this process, + * so falling back to its own (non-empty) skill name is a safe placeholder + * rather than a guess. + */ +function resolveReportSkillExecutionId( + ownId: string | undefined, + skillName: string, + skillExecutionIdByName: Map +): string { + return ownId ?? skillExecutionIdByName.get(skillName) ?? skillName; +} + +function buildFindingObservationsV2( + findingObservations: FindingObservation[], + skillExecutionIdByName: Map +): { observations: FindingObservationV2[]; byOutcome: SummaryV2['byOutcome'] } { + const observations: FindingObservationV2[] = findingObservations.map((observation) => { + const skillExecutionId = resolveObservationSkillExecutionId(observation.skillExecutionId, observation.skill, skillExecutionIdByName); + const origin = { skillExecutionId, skillName: observation.skill ?? '' }; + const findingSnapshot = { + id: observation.finding.id, + reportedId: observation.finding.reportedId, + severity: observation.finding.severity, + confidence: observation.finding.confidence, + title: observation.finding.title, + description: observation.finding.description, + location: observation.finding.location, + elapsedMs: observation.finding.elapsedMs, + }; + + switch (observation.outcome) { + case 'deduped': + return { outcome: 'deduped', origin, finding: findingSnapshot, dedupe: observation.dedupe }; + case 'skipped': + return { outcome: 'skipped', origin, finding: findingSnapshot, skippedReason: observation.skippedReason }; + case 'resolved': + return { outcome: 'resolved', origin, finding: findingSnapshot, resolvedReason: observation.resolvedReason }; + case 'posted': + return { + outcome: 'posted', + origin, + finding: findingSnapshot, + githubCommentId: observation.githubCommentId, + githubCommentUrl: observation.githubCommentUrl, + }; + case 'failed': + return { outcome: 'failed', origin, finding: findingSnapshot }; + } + }); + + const byOutcome = { posted: 0, deduped: 0, skipped: 0, resolved: 0, failed: 0 }; + for (const observation of findingObservations) { + byOutcome[observation.outcome]++; + } + + return { observations, byOutcome }; +} + +interface DedupeAnchor { + existingCommentId?: number; + existingThreadId?: string; +} + +interface HeuristicCorroboratingCandidate { + attribution: FindingAttribution; + targetSkills?: string[]; + anchor: DedupeAnchor; + /** + * Set once this candidate is accepted by some target. A heuristic + * candidate is only ever looked up by its bare `existingFindingId`, which + * two unrelated fresh findings from different skills can coincidentally + * share this run - without this flag, both would independently pass the + * same permissive checks (no anchor on either target, no `targetSkills` + * filter) and both would attach the same corroborator. Claiming makes the + * match at most one target's, at the cost of an arbitrary pick (run order) + * between two truly ambiguous targets - still strictly better than + * corrupting both. + */ + claimed: boolean; +} + +/** + * Split by match basis rather than commingled in one list, so a filter + * written for one basis structurally cannot see candidates of the other. + * `exact` candidates were matched via a dedupe's `existingSkillExecutionId` + * (known whenever the match is against a comment posted earlier in the same + * run) and need no further narrowing - a bare-id collision from an unrelated + * execution physically cannot share that compound key. `heuristic` + * candidates are all a match against a comment from a *prior* run can ever + * produce, since a comment footer only ever records the winner's skill + * *name*, not its skillExecutionId - these still need the name-based + * narrowing `resolveCorroboratingAttributions` applies, plus the anchor + * narrowing below (two prior-run comments can coincidentally share a bare + * model-assigned finding id just as two current-run findings can). + */ +interface CorroboratingCandidates { + exact: Map; + heuristic: Map; + /** + * A resolving finding's own dedupe anchor, keyed by `${skillExecutionId}:${finding.id}` - + * the finding's own internal id, never its bare `existingFindingId`. Two + * findings from the same execution routinely share a bare `existingFindingId` + * (sequential model-assigned continuity ids collide across historical runs), + * so keying on that would let the second overwrite the first's anchor and + * silently drop a real corroborator resolved against the first. + */ + ownAnchors: Map; +} + +function exactCorroborationKey(skillExecutionId: string, findingId: string): string { + return `${skillExecutionId}:${findingId}`; +} + +/** + * Two heuristic candidates only definitely refer to the same prior-run + * comment when both sides carry a comment/thread id and they match - a bare + * `existingFindingId` string alone can't tell two coincidentally-same-id + * prior comments apart, the same collision class `exactCorroborationKey` + * defends against on the current-run side. When either side lacks an + * anchor, there's no way to rule the match out, so it's kept (matches the + * permissive default for unparseable skill metadata). + */ +function anchorsConflict(a: DedupeAnchor, b: DedupeAnchor): boolean { + if (a.existingCommentId !== undefined && b.existingCommentId !== undefined) { + return a.existingCommentId !== b.existingCommentId; + } + if (a.existingThreadId !== undefined && b.existingThreadId !== undefined) { + return a.existingThreadId !== b.existingThreadId; + } + return false; +} + +function buildCorroboratingAttributions( + findingObservations: FindingObservation[], + skillExecutionIdByName: Map +): CorroboratingCandidates { + const exact = new Map(); + const heuristic = new Map(); + const ownAnchors = new Map(); + + for (const observation of findingObservations) { + if (observation.outcome !== 'deduped' || !observation.dedupe.existingFindingId) continue; + + const skillExecutionId = resolveObservationSkillExecutionId(observation.skillExecutionId, observation.skill, skillExecutionIdByName); + const attribution: FindingAttribution = { + skillExecutionId, + skillName: observation.skill ?? '', + role: 'corroborating', + matchType: observation.dedupe.matchType, + }; + const anchor: DedupeAnchor = { + existingCommentId: observation.dedupe.existingCommentId, + existingThreadId: observation.dedupe.existingThreadId, + }; + ownAnchors.set(`${skillExecutionId}:${observation.finding.id}`, anchor); + + if (observation.dedupe.existingSkillExecutionId) { + const key = exactCorroborationKey(observation.dedupe.existingSkillExecutionId, observation.dedupe.existingFindingId); + const list = exact.get(key) ?? []; + list.push(attribution); + exact.set(key, list); + } else { + const key = observation.dedupe.existingFindingId; + const list = heuristic.get(key) ?? []; + list.push({ attribution, targetSkills: observation.dedupe.existingSkills, anchor, claimed: false }); + heuristic.set(key, list); + } + } + + return { exact, heuristic, ownAnchors }; +} + +/** + * Bare finding ids collide across skills, so a heuristic dedupe match + * against `existingFindingId` can name a finding shared by unrelated skills. + * Narrow to the actual winner using the skill(s) recorded on the dedupe + * match, and only trust that narrowing when the target skill name maps to + * exactly one execution in the run - with more than one, there's no way to + * tell which execution a name-only match actually corroborates. + * + * Exact candidates skip all of that: the compound key they were looked up by + * already proves which execution they target, independent of `targetSkillName`. + * + * A bare `existingFindingId` can also collide across two genuinely different + * *prior*-run comments, so heuristic candidates are further narrowed against + * the target's own dedupe anchor (`anchorsConflict`) whenever both sides + * have one - otherwise two unrelated old findings that happen to share a + * model-assigned id could corroborate each other. This narrowing is only as + * good as the target's own anchor: a target posted fresh this run (never + * itself deduped) has none, so a heuristic candidate can't be ruled out for + * it on anchor grounds alone - accepted the same way an empty + * `existingSkills` is (see the "empty skills array" tests below), since + * requiring an anchor here would also block that legitimate case. + * + * Either pass can also dedupe more than one of its own findings against the + * same winner in one run, or (for a finding that dedupes against its own + * prior posting - continuity, not corroboration) recenter onto its own + * skillExecutionId - `seenSkillExecutionIds` collapses/excludes both across + * both passes. + */ +function resolveCorroboratingAttributions( + candidates: CorroboratingCandidates, + findingId: string, + ownFindingId: string, + targetSkillName: string, + targetSkillExecutionId: string, + skillExecutionIdByName: Map +): FindingAttribution[] { + const seenSkillExecutionIds = new Set([targetSkillExecutionId]); + const attributions: FindingAttribution[] = []; + + const exactMatches = candidates.exact.get(exactCorroborationKey(targetSkillExecutionId, findingId)) ?? []; + for (const attribution of exactMatches) { + if (seenSkillExecutionIds.has(attribution.skillExecutionId)) continue; + seenSkillExecutionIds.add(attribution.skillExecutionId); + attributions.push(attribution); + } + + if (skillExecutionIdByName.get(targetSkillName) === targetSkillExecutionId) { + const ownAnchor = candidates.ownAnchors.get(`${targetSkillExecutionId}:${ownFindingId}`); + const heuristicMatches = candidates.heuristic.get(findingId) ?? []; + for (const candidate of heuristicMatches) { + if (candidate.claimed) continue; + if (candidate.targetSkills && candidate.targetSkills.length > 0 && !candidate.targetSkills.includes(targetSkillName)) { + continue; + } + if (ownAnchor && anchorsConflict(ownAnchor, candidate.anchor)) continue; + if (seenSkillExecutionIds.has(candidate.attribution.skillExecutionId)) continue; + seenSkillExecutionIds.add(candidate.attribution.skillExecutionId); + candidate.claimed = true; + attributions.push(candidate.attribution); + } + } + + return attributions; +} + +/** + * Report-time dedupe against an existing comment sets `reportedId` on a + * finding (see `syncReportedIds` in poster.ts) without ever changing its + * `id`. The analyze-phase payload being patched here was built before that + * happened, so newly-discovered `reportedId`s need folding in. `id` alone + * is only unique within one skill execution, not across the whole run, so + * the map is keyed by `${skillExecutionId}:${id}` - the same composite key + * `buildReportModeResultsV2` already uses for this exact reason. + */ +function buildReportedIdMap( + findingObservations: FindingObservation[], + skillExecutionIdByName: Map +): Map { + const reportedIds = new Map(); + for (const observation of findingObservations) { + if (observation.outcome !== 'deduped' || !observation.finding.reportedId) continue; + const skillExecutionId = resolveObservationSkillExecutionId(observation.skillExecutionId, observation.skill, skillExecutionIdByName); + reportedIds.set(`${skillExecutionId}:${observation.finding.id}`, observation.finding.reportedId); + } + return reportedIds; +} + +interface GithubCommentRef { + githubCommentId?: number; + githubCommentUrl?: string; +} + +/** + * `githubCommentId`/`githubCommentUrl` only exist on the `posted` variant of + * a `FindingObservation` (they come back from the GitHub API at posting + * time, unlike `reportedId` which `syncReportedIds` writes onto the `Finding` + * object itself) - so, unlike reportedId, this has no live-object fallback + * and must always be looked up from `findingObservations`. + */ +function buildGithubCommentMap( + findingObservations: FindingObservation[], + skillExecutionIdByName: Map +): Map { + const commentsById = new Map(); + for (const observation of findingObservations) { + if (observation.outcome !== 'posted') continue; + if (!observation.githubCommentId && !observation.githubCommentUrl) continue; + const skillExecutionId = resolveObservationSkillExecutionId(observation.skillExecutionId, observation.skill, skillExecutionIdByName); + commentsById.set(`${skillExecutionId}:${observation.finding.id}`, { + githubCommentId: observation.githubCommentId, + githubCommentUrl: observation.githubCommentUrl, + }); + } + return commentsById; +} + +interface SkillExecutionUsageUpdate { + usage: SkillExecution['usage']; + auxiliaryUsage: SkillExecution['auxiliaryUsage']; + checkRunUrl: SkillExecution['checkRunUrl']; + checkRunId: SkillExecution['checkRunId']; + reviewEvent: SkillExecution['reviewEvent']; + checkConclusion: SkillExecution['checkConclusion']; +} + +/** + * Report-phase posting (dedupe/consolidate) merges auxiliary usage onto + * `result.report` (see poster.ts) after the analyze-phase payload being + * patched here was already built. Unlike verification/merge provenance, + * this isn't data that only `findingProcessingEvents` can reconstruct - + * `results` carries the live, already-mutated report at patch time, the + * same source v1's write reads from, so it can be folded in directly. The + * same is true of `checkRunUrl`/`checkRunId`: report mode creates its skill + * checks as already-completed check runs (`createCompletedSkillChecksForReport`) + * only after the analyze-phase payload was built, so `result` is the only + * place this run's real check identity exists. `reviewEvent`/`checkConclusion` + * follow the same rule: dedup can shrink the finding set posted (or posting + * can fail/get blocked) after the analyze-phase payload was built, so only + * `result` at patch time reflects what was actually posted and concluded. + */ +function buildSkillExecutionUsageUpdates( + results: TriggerResult[], + triggerById: Map, + skillExecutionIdByName: Map +): Map { + const updates = new Map(); + + for (const result of results) { + const report = result.report; + if (!report) continue; + + const trigger = result.triggerId ? triggerById.get(result.triggerId) : undefined; + const skillExecutionId = resolveReportSkillExecutionId(trigger?.skillExecutionId, report.skill, skillExecutionIdByName); + const auxiliaryUsageEntries = toAuxiliaryUsageEntries(report.auxiliaryUsage, report.auxiliaryUsageAttribution); + + updates.set(skillExecutionId, { + usage: report.usage, + auxiliaryUsage: auxiliaryUsageEntries.length > 0 ? auxiliaryUsageEntries : undefined, + checkRunUrl: result.checkRunUrl, + checkRunId: result.checkRunId, + reviewEvent: result.renderResult?.review?.event, + checkConclusion: determineConclusion( + filterFindings(report.findings, undefined, result.minConfidence), + result.failOn, + result.failCheck + ), + }); + } + + return updates; +} + +/** + * Rebuild only the observation-derived parts of a v2 findings payload: + * `findingObservations`, `summary.byOutcome`, any newly-discovered + * `reportedId` (continuity with an existing comment), any newly-discovered + * cross-skill corroboration on `findings[].reportedBy`, and each skill + * execution's `usage`/`auxiliaryUsage` (report-phase posting costs). Used + * by report mode to fold real posting outcomes into an analyze-phase + * payload without touching `skillExecutions[].findingIds`/verification/ + * merge provenance or `discardedFindings`, which can only be reconstructed + * from the original `findingProcessingEvents` and would otherwise be + * silently wiped by a full rebuild from replayed results. Corroboration is + * additive-only (existing `reportedBy` entries are never removed) since it + * can only be discovered once posting/dedup runs, which analyze mode never + * does. + */ +export function patchFindingsOutputV2Observations( + base: WardenFindingsV2, + results: TriggerResult[], + matchedTriggers: ResolvedTrigger[], + findingObservations: FindingObservation[] +): WardenFindingsV2 { + const triggerById = new Map(matchedTriggers.map((t) => [t.id, t])); + const skillExecutionIdByName = skillExecutionIdByNameFrom(matchedTriggers); + const { observations, byOutcome } = buildFindingObservationsV2(findingObservations, skillExecutionIdByName); + const corroboratingById = buildCorroboratingAttributions(findingObservations, skillExecutionIdByName); + const reportedIdMap = buildReportedIdMap(findingObservations, skillExecutionIdByName); + const githubCommentMap = buildGithubCommentMap(findingObservations, skillExecutionIdByName); + const usageUpdates = buildSkillExecutionUsageUpdates(results, triggerById, skillExecutionIdByName); + + const skillExecutions = base.skillExecutions.map((execution) => { + const update = usageUpdates.get(execution.skillExecutionId); + if (!update) return execution; + return { + ...execution, + usage: update.usage, + auxiliaryUsage: update.auxiliaryUsage, + checkRunUrl: update.checkRunUrl, + checkRunId: update.checkRunId, + reviewEvent: update.reviewEvent, + checkConclusion: update.checkConclusion, + }; + }); + + const findings = base.findings.map((original) => { + const originSkillExecutionId = original.reportedBy.find((r) => r.role === 'primary')?.skillExecutionId ?? ''; + const reportedId = reportedIdMap.get(`${originSkillExecutionId}:${original.id}`); + const githubComment = githubCommentMap.get(`${originSkillExecutionId}:${original.id}`); + const finding = { + ...original, + ...(reportedId && { reportedId }), + ...(githubComment && { + githubCommentId: githubComment.githubCommentId, + githubCommentUrl: githubComment.githubCommentUrl, + }), + }; + + const primarySkillName = finding.reportedBy.find((r) => r.role === 'primary')?.skillName ?? ''; + const newCorroborators = resolveCorroboratingAttributions( + corroboratingById, + displayFindingId(finding), + finding.id, + primarySkillName, + originSkillExecutionId, + skillExecutionIdByName + ); + if (newCorroborators.length === 0) return finding; + + const existingSkillExecutionIds = new Set(finding.reportedBy.map((r) => r.skillExecutionId)); + const additions = newCorroborators.filter((c) => !existingSkillExecutionIds.has(c.skillExecutionId)); + if (additions.length === 0) return finding; + + return { ...finding, reportedBy: [...finding.reportedBy, ...additions] }; + }); + + // Every field is named explicitly (no `...base` spread) so a new field + // added to WardenFindingsV2 fails to compile here until this function + // decides whether report mode owns it or must pass it through from base. + const patched: WardenFindingsV2 = { + schemaVersion: base.schemaVersion, + runId: base.runId, + skillExecutions, + discardedFindings: base.discardedFindings, + findings, + findingObservations: observations, + summary: { + totalFindings: base.summary.totalFindings, + totalSkillExecutions: base.summary.totalSkillExecutions, + bySeverity: base.summary.bySeverity, + byOutcome, + }, + }; + return WardenFindingsSchemaV2.parse(patched); +} + +export interface BuildFindingsOutputV2Options { + runId: string; +} + +interface ReducedFindingProcessingEvents { + verificationById: Map; + mergeById: Map; + discarded: DiscardedFinding[]; +} + +/** Finding IDs are model-assigned per skill run and can collide across skills, so the returned maps must not survive past this execution's findings. */ +function reduceFindingProcessingEvents( + events: TriggerResult['findingProcessingEvents'], + skillExecutionId: string +): ReducedFindingProcessingEvents { + const verificationById = new Map(); + const mergeById = new Map(); + const discarded: DiscardedFinding[] = []; + + for (const event of events ?? []) { + if (event.stage === 'verification' && event.action === 'revised' && event.replacement) { + verificationById.set(event.replacement.id, { + outcome: 'revised', + model: event.model, + runtime: event.runtime, + evidence: event.replacement.verification, + before: { + title: event.finding.title, + description: event.finding.description, + severity: event.finding.severity, + confidence: event.finding.confidence, + }, + }); + } else if (event.stage === 'verification' && event.action === 'kept') { + verificationById.set(event.finding.id, { + outcome: 'kept', + model: event.model, + runtime: event.runtime, + }); + } else if (event.stage === 'verification' && event.action === 'rejected') { + discarded.push({ + originSkillExecutionId: skillExecutionId, + stage: 'verification_rejected', + severity: event.finding.severity, + title: event.finding.title, + location: event.finding.location, + model: event.model, + reason: event.reason, + }); + } else if (event.stage === 'merge' && event.action === 'merged') { + const survivorId = event.replacement?.id; + discarded.push({ + originSkillExecutionId: skillExecutionId, + stage: 'merge_absorbed', + severity: event.finding.severity, + title: event.finding.title, + location: event.finding.location, + model: event.model, + reason: event.reason, + survivorFindingId: survivorId, + }); + if (survivorId) { + const entry = mergeById.get(survivorId) ?? { model: event.model, runtime: event.runtime, absorbedFindingIds: [] }; + entry.absorbedFindingIds.push(event.finding.id); + mergeById.set(survivorId, entry); + } + } + } + + return { verificationById, mergeById, discarded }; +} + +/** Build the schema-v2 findings output from scratch: skill executions, findings, corroboration, and discarded findings from this run's results. */ +export function buildFindingsOutputV2( + results: TriggerResult[], + matchedTriggers: ResolvedTrigger[], + findingObservations: FindingObservation[], + options: BuildFindingsOutputV2Options +): WardenFindingsV2 { + const triggerById = new Map(matchedTriggers.map((t) => [t.id, t])); + const skillExecutionIdByName = skillExecutionIdByNameFrom(matchedTriggers); + const corroboratingById = buildCorroboratingAttributions(findingObservations, skillExecutionIdByName); + const githubCommentMap = buildGithubCommentMap(findingObservations, skillExecutionIdByName); + + const skillExecutions: SkillExecution[] = []; + const findings: ExportedFindingV2[] = []; + const discardedFindings: DiscardedFinding[] = []; + + for (const result of results) { + const report = result.report; + if (!report) continue; + + const trigger = result.triggerId ? triggerById.get(result.triggerId) : undefined; + const skillExecutionId = resolveReportSkillExecutionId(trigger?.skillExecutionId, report.skill, skillExecutionIdByName); + + const { verificationById, mergeById, discarded } = reduceFindingProcessingEvents( + result.findingProcessingEvents, + skillExecutionId + ); + discardedFindings.push(...discarded); + + const auxiliaryUsageEntries = toAuxiliaryUsageEntries(report.auxiliaryUsage, report.auxiliaryUsageAttribution); + + skillExecutions.push({ + skillExecutionId, + skillName: report.skill, + triggerId: result.triggerId, + triggerName: result.triggerName, + model: report.model, + models: report.models, + runtime: report.runtime, + auxiliaryModel: trigger?.auxiliaryModel, + synthesisModel: trigger?.synthesisModel, + summary: report.summary, + durationMs: report.durationMs, + usage: report.usage, + auxiliaryUsage: auxiliaryUsageEntries.length > 0 ? auxiliaryUsageEntries : undefined, + findingsBySeverity: severityBreakdown(report.findings), + findingIds: report.findings.map((f) => f.id), + failedHunks: report.failedHunks, + failedExtractions: report.failedExtractions, + error: report.error, + verifierRejections: report.verifierRejections, + checkRunUrl: result.checkRunUrl, + checkRunId: result.checkRunId, + reviewEvent: result.renderResult?.review?.event, + checkConclusion: determineConclusion( + filterFindings(report.findings, undefined, result.minConfidence), + result.failOn, + result.failCheck + ), + issueNumber: result.issueNumber, + issueUrl: result.issueUrl, + }); + + for (const finding of report.findings) { + const githubComment = githubCommentMap.get(`${skillExecutionId}:${finding.id}`); + findings.push({ + id: finding.id, + reportedId: finding.reportedId, + contentHash: generateContentHash(finding.title, finding.description), + severity: finding.severity, + confidence: finding.confidence, + title: finding.title, + description: finding.description, + verification: finding.verification, + location: finding.location, + additionalLocations: finding.additionalLocations, + sourceSnippet: finding.sourceSnippet, + reportedBy: [ + { skillExecutionId, skillName: report.skill, role: 'primary' }, + ...resolveCorroboratingAttributions( + corroboratingById, + displayFindingId(finding), + finding.id, + report.skill, + skillExecutionId, + skillExecutionIdByName + ), + ], + provenance: { + originSkillExecutionId: skillExecutionId, + originModel: report.model, + verification: verificationById.get(finding.id), + merge: mergeById.get(finding.id), + }, + githubCommentId: githubComment?.githubCommentId, + githubCommentUrl: githubComment?.githubCommentUrl, + }); + } + } + + const { observations, byOutcome } = buildFindingObservationsV2(findingObservations, skillExecutionIdByName); + + return WardenFindingsSchemaV2.parse({ + schemaVersion: '2', + runId: options.runId, + skillExecutions, + findings, + discardedFindings: discardedFindings.length > 0 ? discardedFindings : undefined, + findingObservations: observations, + summary: { + totalFindings: findings.length, + totalSkillExecutions: skillExecutions.length, + bySeverity: severityBreakdown(findings), + byOutcome, + }, + }); +} + +export function toFindingFromV2(finding: ExportedFindingV2): Finding { + return { + id: finding.id, + reportedId: finding.reportedId, + severity: finding.severity, + confidence: finding.confidence, + title: finding.title, + description: finding.description, + verification: finding.verification, + location: finding.location, + additionalLocations: finding.additionalLocations, + sourceSnippet: finding.sourceSnippet, + }; +} + +/** + * Rebuild the `SkillReport[]` a v2 artifact pair describes, independent of any + * currently-configured triggers. Used for CLI replay (`warden runs show`), + * where the goal is "show me what this run produced," not + * `buildReportModeResultsV2`'s "should I re-post this against current config." + */ +export function reconstructSkillReportsFromV2(findingsOutput: WardenFindingsV2): SkillReport[] { + const findingsByExecutionScopedId = new Map( + findingsOutput.findings.map((finding) => [`${finding.provenance.originSkillExecutionId}:${finding.id}`, finding]) + ); + + return findingsOutput.skillExecutions.map((execution) => { + const findings = execution.findingIds.flatMap((id) => { + const finding = findingsByExecutionScopedId.get(`${execution.skillExecutionId}:${id}`); + return finding ? [toFindingFromV2(finding)] : []; + }); + + const { usage: auxiliaryUsage, attribution: auxiliaryUsageAttribution } = fromAuxiliaryUsageEntries( + execution.auxiliaryUsage + ); + + return { + skill: execution.skillName, + summary: execution.summary, + findings, + durationMs: execution.durationMs, + usage: execution.usage, + auxiliaryUsage, + auxiliaryUsageAttribution, + failedHunks: execution.failedHunks, + failedExtractions: execution.failedExtractions, + error: execution.error, + verifierRejections: execution.verifierRejections, + model: execution.model, + models: execution.models, + runtime: execution.runtime, + }; + }); +} diff --git a/packages/warden/src/action/reporting/output.test.ts b/packages/warden/src/action/reporting/output.test.ts index 7ced469bd..ed126d513 100644 --- a/packages/warden/src/action/reporting/output.test.ts +++ b/packages/warden/src/action/reporting/output.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { EventContext, Finding, SkillReport } from '../../types/index.js'; -import { buildFindingsOutput, FindingsOutputSchema } from './output.js'; +import { buildConfiguredSkillsList, buildFindingsOutput, FindingsOutputSchema } from './output.js'; describe('findings output schema', () => { it('builds a schema-valid public findings payload', () => { @@ -23,6 +23,69 @@ describe('findings output schema', () => { }); }); + it('shares a join key across skills[], findingObservations[], and triggerResults[] for a deduped finding', () => { + const finding = { ...createFinding(), reportedId: 'EXISTING-001' }; + const report = createReport({ findings: [finding] }); + + const output = buildFindingsOutput([report], createContext(), [ + { + outcome: 'deduped', + finding, + skill: 'test-skill', + dedupe: { source: 'warden', matchType: 'hash', existingFindingId: 'EXISTING-001' }, + }, + ], { + triggerResults: [{ triggerName: 'test-trigger', skillName: 'test-skill', report }], + }); + + expect(output.skills[0]?.findings[0]?.id).toBe('EXISTING-001'); + expect(output.findingObservations[0]?.finding.id).toBe('EXISTING-001'); + const triggerResult = output.triggerResults?.[0]; + expect(triggerResult?.status).toBe('success'); + expect(triggerResult?.status === 'success' ? triggerResult.report.findings[0]?.id : undefined).toBe('EXISTING-001'); + }); + + it('never includes schema-v2-only fields on findingObservations, regardless of what the runtime observation carries', () => { + const postedFinding = { ...createFinding(), id: 'POSTED-001' }; + const dedupedFinding = { ...createFinding(), id: 'DEDUPED-001' }; + + const output = buildFindingsOutput([createReport({ findings: [postedFinding, dedupedFinding] })], createContext(), [ + { + outcome: 'posted', + finding: postedFinding, + skill: 'test-skill', + skillExecutionId: 'exec-1', + githubCommentId: 42, + githubCommentUrl: 'https://github.com/example/pull/1#discussion_r42', + }, + { + outcome: 'deduped', + finding: dedupedFinding, + skill: 'test-skill', + skillExecutionId: 'exec-1', + dedupe: { + source: 'warden', + matchType: 'hash', + existingFindingId: 'DEDUPED-001', + existingSkillExecutionId: 'exec-0', + existingSkills: ['test-skill', 'other-skill'], + }, + }, + ]); + + expect(FindingsOutputSchema.parse(output)).toEqual(output); + expect(output.findingObservations[0]).not.toHaveProperty('skillExecutionId'); + expect(output.findingObservations[0]).not.toHaveProperty('githubCommentId'); + expect(output.findingObservations[0]).not.toHaveProperty('githubCommentUrl'); + expect(output.findingObservations[1]).not.toHaveProperty('skillExecutionId'); + const deduped = output.findingObservations[1]; + expect(deduped?.outcome === 'deduped' ? deduped.dedupe : undefined).toEqual({ + source: 'warden', + matchType: 'hash', + existingFindingId: 'DEDUPED-001', + }); + }); + it('includes trigger run results for split report mode', () => { const report = createReport(); const output = buildFindingsOutput([report], createContext(), [], { @@ -89,6 +152,9 @@ describe('findings output schema', () => { metadata: { internal: true }, runtime: 'pi', failedHunks: 1, + failedExtractions: 2, + error: { code: 'sdk_error', message: 'partial failure' }, + verifierRejections: { count: 1, reasons: ['not reproducible'] }, }); const output = buildFindingsOutput([report], createContext(), [], { timestamp: '2026-01-01T00:00:00.000Z', @@ -107,11 +173,14 @@ describe('findings output schema', () => { report: { skill: 'test-skill', summary: 'Found 1 issue', + failedHunks: 1, + failedExtractions: 2, + error: { code: 'sdk_error', message: 'partial failure' }, + verifierRejections: { count: 1, reasons: ['not reproducible'] }, }, }); expect(output.triggerResults?.[0]).not.toHaveProperty('report.metadata'); expect(output.triggerResults?.[0]).not.toHaveProperty('report.runtime'); - expect(output.triggerResults?.[0]).not.toHaveProperty('report.failedHunks'); }); it('requires status-specific trigger result data', () => { @@ -167,6 +236,30 @@ describe('findings output schema', () => { ).toThrow(); }); + it('includes verifierRejections when present', () => { + const report = createReport({ + verifierRejections: { count: 1, reasons: ['not reproducible'] }, + }); + const output = buildFindingsOutput([report], createContext(), [], { + timestamp: '2026-01-01T00:00:00.000Z', + runId: '123', + }); + + expect(FindingsOutputSchema.parse(output)).toEqual(output); + expect(output.skills[0]).toMatchObject({ + verifierRejections: { count: 1, reasons: ['not reproducible'] }, + }); + }); + + it('omits verifierRejections when absent', () => { + const output = buildFindingsOutput([createReport()], createContext(), [], { + timestamp: '2026-01-01T00:00:00.000Z', + runId: '123', + }); + + expect(output.skills[0]?.verifierRejections).toBeUndefined(); + }); + it('rejects sentinel dedupe comment IDs', () => { const output = buildFindingsOutput([createReport()], createContext(), [], { timestamp: '2026-01-01T00:00:00.000Z', @@ -227,6 +320,68 @@ describe('findings output schema', () => { expect('failedExtractions' in serialized).toBe(false); expect('error' in serialized).toBe(false); }); + + it('includes the configured skills roster when provided', () => { + const output = buildFindingsOutput([createReport()], createContext(), [], { + timestamp: '2026-01-01T00:00:00.000Z', + runId: '123', + configuredSkills: [ + { name: 'test-skill', triggered: true }, + { name: 'idle-skill', triggered: false }, + ], + }); + + expect(FindingsOutputSchema.parse(output)).toEqual(output); + expect(output.configuredSkills).toEqual([ + { name: 'test-skill', triggered: true }, + { name: 'idle-skill', triggered: false }, + ]); + }); + + it('omits the configured skills roster when not provided', () => { + const output = buildFindingsOutput([createReport()], createContext(), [], { + timestamp: '2026-01-01T00:00:00.000Z', + runId: '123', + }); + + expect(output.configuredSkills).toBeUndefined(); + }); +}); + +describe('buildConfiguredSkillsList', () => { + it('marks matched skills as triggered and unmatched skills as not', () => { + const result = buildConfiguredSkillsList({ + allTriggers: [{ name: 'matched-skill' }, { name: 'skipped-skill' }], + matchedTriggers: [{ name: 'matched-skill' }], + }); + + expect(result).toEqual([ + { name: 'matched-skill', triggered: true }, + { name: 'skipped-skill', triggered: false }, + ]); + }); + + it('deduplicates multiple trigger blocks for the same skill', () => { + const result = buildConfiguredSkillsList({ + allTriggers: [{ name: 'multi-trigger-skill' }, { name: 'multi-trigger-skill' }], + matchedTriggers: [{ name: 'multi-trigger-skill' }], + }); + + expect(result).toEqual([{ name: 'multi-trigger-skill', triggered: true }]); + }); + + it('returns an empty list when nothing is configured', () => { + expect(buildConfiguredSkillsList({ allTriggers: [], matchedTriggers: [] })).toEqual([]); + }); + + it('includes a skill whose only trigger is neither matched nor a PR-check skip', () => { + const result = buildConfiguredSkillsList({ + allTriggers: [{ name: 'nightly-sweep' }], + matchedTriggers: [], + }); + + expect(result).toEqual([{ name: 'nightly-sweep', triggered: false }]); + }); }); function createFinding(): Finding { diff --git a/packages/warden/src/action/reporting/output.ts b/packages/warden/src/action/reporting/output.ts index 2cce40cb0..eeb34d2af 100644 --- a/packages/warden/src/action/reporting/output.ts +++ b/packages/warden/src/action/reporting/output.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import type { EventContext, SkillReport } from '../../types/index.js'; +import type { EventContext, Finding, SkillReport } from '../../types/index.js'; import { AuxiliaryUsageMapSchema, FindingSchema, @@ -8,9 +8,11 @@ import { SkillErrorSchema, SourceSnippetSchema, UsageStatsSchema, + VerifierRejectionsSchema, } from '../../types/index.js'; import type { FindingObservation } from './outcomes.js'; import { FindingObservationSchema } from './outcomes.js'; +import { displayFindingId } from '../../cli/output/formatters.js'; const ExportedFindingSchema = z.object({ id: z.string(), @@ -45,6 +47,10 @@ const ReplaySkillReportSchema = z.object({ usage: UsageStatsSchema.optional(), auxiliaryUsage: AuxiliaryUsageMapSchema.optional(), model: z.string().optional(), + failedHunks: z.number().int().nonnegative().optional(), + failedExtractions: z.number().int().nonnegative().optional(), + error: SkillErrorSchema.optional(), + verifierRejections: VerifierRejectionsSchema.optional(), }); export const TriggerRunResultSchema = z.discriminatedUnion('status', [ @@ -96,10 +102,15 @@ export const FindingsOutputSchema = z.object({ failedHunks: z.number().int().nonnegative().optional(), failedExtractions: z.number().int().nonnegative().optional(), error: SkillErrorSchema.optional(), + verifierRejections: VerifierRejectionsSchema.optional(), findings: z.array(ExportedFindingSchema), })), triggerResults: z.array(TriggerRunResultSchema).optional(), findingObservations: z.array(FindingObservationSchema), + configuredSkills: z.array(z.object({ + name: z.string(), + triggered: z.boolean(), + })).optional(), }); export type FindingsOutput = z.infer; @@ -116,9 +127,31 @@ interface BuildFindingsOutputOptions { timestamp?: string; runId?: string; triggerResults?: ReplayTriggerResult[]; + configuredSkills?: { name: string; triggered: boolean }[]; +} + +/** Build the configured-skills roster, deduping by name since a skill's multiple trigger blocks (e.g. PR + schedule) share one name. */ +export function buildConfiguredSkillsList({ + allTriggers, + matchedTriggers, +}: { + allTriggers: { name: string }[]; + matchedTriggers: { name: string }[]; +}): { name: string; triggered: boolean }[] { + const matchedNames = new Set(matchedTriggers.map((t) => t.name)); + const seen = new Set(); + const result: { name: string; triggered: boolean }[] = []; + + for (const trigger of allTriggers) { + if (seen.has(trigger.name)) continue; + seen.add(trigger.name); + result.push({ name: trigger.name, triggered: matchedNames.has(trigger.name) }); + } + + return result; } -function serializeTriggerError(error: unknown): z.infer { +export function serializeTriggerError(error: unknown): z.infer { if (error instanceof Error) { return { name: error.name, @@ -133,11 +166,15 @@ function serializeReplayReport(report: SkillReport): z.infer ({ ...f, id: v1ContinuityId(f) })), durationMs: report.durationMs, usage: report.usage, auxiliaryUsage: report.auxiliaryUsage, model: report.model, + failedHunks: report.failedHunks, + failedExtractions: report.failedExtractions, + error: report.error, + verifierRejections: report.verifierRejections, }; } @@ -161,6 +198,53 @@ function serializeTriggerResult(result: ReplayTriggerResult): z.infer { + const finding = { ...observation.finding, id: v1ContinuityId(observation.finding) }; + + switch (observation.outcome) { + case 'posted': + return { outcome: 'posted', finding, skill: observation.skill }; + case 'deduped': + return { + outcome: 'deduped', + finding, + skill: observation.skill, + dedupe: { + source: observation.dedupe.source, + matchType: observation.dedupe.matchType, + existingFindingId: observation.dedupe.existingFindingId, + existingCommentId: observation.dedupe.existingCommentId, + existingThreadId: observation.dedupe.existingThreadId, + existingResolved: observation.dedupe.existingResolved, + actor: observation.dedupe.actor, + }, + }; + case 'skipped': + return { outcome: 'skipped', finding, skill: observation.skill, skippedReason: observation.skippedReason }; + case 'resolved': + return { outcome: 'resolved', finding, skill: observation.skill, resolvedReason: observation.resolvedReason }; + case 'failed': + return { outcome: 'failed', finding, skill: observation.skill }; + } +} + /** Build the public findings export payload. */ export function buildFindingsOutput( reports: SkillReport[], @@ -207,8 +291,9 @@ export function buildFindingsOutput( failedHunks: r.failedHunks, failedExtractions: r.failedExtractions, error: r.error, + verifierRejections: r.verifierRejections, findings: r.findings.map((f) => ({ - id: f.id, + id: v1ContinuityId(f), severity: f.severity, confidence: f.confidence, title: f.title, @@ -221,7 +306,8 @@ export function buildFindingsOutput( ...(options.triggerResults && { triggerResults: options.triggerResults.map(serializeTriggerResult), }), - findingObservations, + findingObservations: findingObservations.map(serializeV1FindingObservation), + ...(options.configuredSkills && { configuredSkills: options.configuredSkills }), }; return FindingsOutputSchema.parse(output); diff --git a/packages/warden/src/action/review/poster.test.ts b/packages/warden/src/action/review/poster.test.ts index 504f9caba..a6319098a 100644 --- a/packages/warden/src/action/review/poster.test.ts +++ b/packages/warden/src/action/review/poster.test.ts @@ -39,6 +39,7 @@ describe('postTriggerReview', () => { vi.mocked(mockOctokit.pulls.createReview).mockResolvedValue({} as never); vi.mocked(mockOctokit.pulls.get).mockResolvedValue({ data: { head: { sha: 'abc123' } } } as never); + vi.mocked(mockOctokit.paginate).mockResolvedValue([]); mockDeps = { octokit: mockOctokit, context: mockContext, @@ -47,9 +48,11 @@ describe('postTriggerReview', () => { }); const mockOctokit = { + paginate: vi.fn().mockResolvedValue([]), pulls: { createReview: vi.fn().mockResolvedValue({}), get: vi.fn().mockResolvedValue({ data: { head: { sha: 'abc123' } } }), + listCommentsForReview: vi.fn(), }, } as unknown as Octokit; @@ -244,6 +247,230 @@ describe('postTriggerReview', () => { }); }); + it('attaches the real comment id/url once GitHub returns the posted review', async () => { + const finding = createFinding(); + const result: TriggerResult = { + triggerName: 'test-trigger', + skillName: 'test-skill', + report: { + skill: 'test-skill', + summary: 'Found 1 issue', + findings: [finding], + usage: { inputTokens: 100, outputTokens: 50, costUSD: 0.01 }, + }, + renderResult: createRenderResult({ + review: { + event: 'COMMENT', + body: 'Test review', + comments: [{ path: 'test.ts', line: 10, body: 'Test comment', findingId: finding.id }], + }, + }), + reportOn: 'low', + }; + + vi.mocked(findingToExistingComment).mockReturnValue(createExistingComment()); + vi.mocked(mockOctokit.pulls.createReview).mockResolvedValueOnce({ data: { id: 555 } } as never); + vi.mocked(mockOctokit.paginate).mockResolvedValueOnce([ + { + id: 987, + html_url: 'https://github.com/test-owner/test-repo/pull/1#discussion_r987', + body: 'Test comment', + path: 'test.ts', + line: 10, + }, + ]); + + const postResult = await postTriggerReview({ + result, + existingComments: [], + apiKey: 'test-key', + }, mockDeps); + + expect(mockOctokit.paginate).toHaveBeenCalledWith( + mockOctokit.pulls.listCommentsForReview, + expect.objectContaining({ owner: 'test-owner', repo: 'test-repo', pull_number: 1, review_id: 555 }) + ); + expect(postResult.findingObservations).toEqual([ + expect.objectContaining({ + outcome: 'posted', + finding, + githubCommentId: 987, + githubCommentUrl: 'https://github.com/test-owner/test-repo/pull/1#discussion_r987', + }), + ]); + }); + + it('matches every comment on a review spanning more than one GitHub results page', async () => { + const findingCount = 150; + const findings = Array.from({ length: findingCount }, (_, i) => + createFinding({ id: `test-${i}`, location: { path: 'test.ts', startLine: 10 + i } }) + ); + const result: TriggerResult = { + triggerName: 'test-trigger', + skillName: 'test-skill', + report: { + skill: 'test-skill', + summary: `Found ${findingCount} issues`, + findings, + usage: { inputTokens: 100, outputTokens: 50, costUSD: 0.01 }, + }, + renderResult: createRenderResult({ + review: { + event: 'COMMENT', + body: 'Test review', + comments: findings.map((finding) => ({ + path: 'test.ts', + line: finding.location!.startLine, + body: `Comment for ${finding.id}`, + findingId: finding.id, + })), + }, + }), + reportOn: 'low', + }; + + vi.mocked(findingToExistingComment).mockReturnValue(createExistingComment()); + vi.mocked(mockOctokit.pulls.createReview).mockResolvedValueOnce({ data: { id: 555 } } as never); + vi.mocked(mockOctokit.paginate).mockResolvedValueOnce( + findings.map((finding, i) => ({ + id: 1000 + i, + html_url: `https://github.com/test-owner/test-repo/pull/1#discussion_r${1000 + i}`, + body: `Comment for ${finding.id}`, + path: 'test.ts', + line: finding.location!.startLine, + })) + ); + + const postResult = await postTriggerReview({ + result, + existingComments: [], + apiKey: 'test-key', + }, mockDeps); + + expect(mockOctokit.paginate).toHaveBeenCalledWith( + mockOctokit.pulls.listCommentsForReview, + expect.objectContaining({ per_page: 100 }) + ); + expect(postResult.findingObservations).toHaveLength(findingCount); + expect(postResult.findingObservations.every((obs) => 'githubCommentId' in obs && obs.githubCommentId !== undefined)).toBe(true); + }); + + it('matches identically-worded comments to the right posted id by location, not body order', async () => { + const findingA = createFinding({ id: 'finding-a', location: { path: 'a.ts', startLine: 10 } }); + const findingB = createFinding({ id: 'finding-b', location: { path: 'b.ts', startLine: 20 } }); + const result: TriggerResult = { + triggerName: 'test-trigger', + skillName: 'test-skill', + report: { + skill: 'test-skill', + summary: 'Found 2 issues', + findings: [findingA, findingB], + usage: { inputTokens: 100, outputTokens: 50, costUSD: 0.01 }, + }, + renderResult: createRenderResult({ + review: { + event: 'COMMENT', + body: 'Test review', + comments: [ + { path: 'a.ts', line: 10, body: 'Duplicate wording', findingId: findingA.id }, + { path: 'b.ts', line: 20, body: 'Duplicate wording', findingId: findingB.id }, + ], + }, + }), + reportOn: 'low', + }; + + vi.mocked(findingToExistingComment).mockReturnValue(createExistingComment()); + vi.mocked(mockOctokit.pulls.createReview).mockResolvedValueOnce({ data: { id: 555 } } as never); + vi.mocked(mockOctokit.paginate).mockResolvedValueOnce([ + { id: 200, html_url: 'https://github.com/test-owner/test-repo/pull/1#discussion_r200', body: 'Duplicate wording', path: 'b.ts', line: 20 }, + { id: 100, html_url: 'https://github.com/test-owner/test-repo/pull/1#discussion_r100', body: 'Duplicate wording', path: 'a.ts', line: 10 }, + ]); + + const postResult = await postTriggerReview({ + result, + existingComments: [], + apiKey: 'test-key', + }, mockDeps); + + expect(postResult.findingObservations).toEqual([ + expect.objectContaining({ finding: findingA, githubCommentId: 100, githubCommentUrl: expect.stringContaining('r100') }), + expect.objectContaining({ finding: findingB, githubCommentId: 200, githubCommentUrl: expect.stringContaining('r200') }), + ]); + }); + + it('still reports the finding as posted when fetching the real comment id fails', async () => { + const finding = createFinding(); + const result: TriggerResult = { + triggerName: 'test-trigger', + skillName: 'test-skill', + report: { + skill: 'test-skill', + summary: 'Found 1 issue', + findings: [finding], + usage: { inputTokens: 100, outputTokens: 50, costUSD: 0.01 }, + }, + renderResult: createRenderResult({ + review: { + event: 'COMMENT', + body: 'Test review', + comments: [{ path: 'test.ts', line: 10, body: 'Test comment', findingId: finding.id }], + }, + }), + reportOn: 'low', + }; + + vi.mocked(findingToExistingComment).mockReturnValue(createExistingComment()); + vi.mocked(mockOctokit.pulls.createReview).mockResolvedValueOnce({ data: { id: 555 } } as never); + vi.mocked(mockOctokit.paginate).mockRejectedValueOnce(new Error('rate limited')); + + const postResult = await postTriggerReview({ + result, + existingComments: [], + apiKey: 'test-key', + }, mockDeps); + + expect(postResult.posted).toBe(true); + expect(postResult.findingObservations).toEqual([ + expect.objectContaining({ outcome: 'posted', finding, githubCommentId: undefined, githubCommentUrl: undefined }), + ]); + }); + + it('stamps the posted observation with the trigger result\'s own skillExecutionId', async () => { + const finding = createFinding(); + const result: TriggerResult = { + triggerName: 'test-trigger', + skillName: 'test-skill', + skillExecutionId: 'exec-42', + report: { + skill: 'test-skill', + summary: 'Found 1 issue', + findings: [finding], + usage: { inputTokens: 100, outputTokens: 50, costUSD: 0.01 }, + }, + renderResult: createRenderResult({ + review: { + event: 'COMMENT', + body: 'Test review', + comments: [{ path: 'test.ts', line: 10, body: 'Test comment' }], + }, + }), + reportOn: 'low', + }; + + vi.mocked(findingToExistingComment).mockReturnValue(createExistingComment()); + + const postResult = await postTriggerReview({ + result, + existingComments: [], + apiKey: 'test-key', + }, mockDeps); + + expect(postResult.findingObservations).toEqual([ + { outcome: 'posted', finding, skill: 'test-skill', skillExecutionId: 'exec-42' }, + ]); + }); + it('skips body-only non-blocking reviews', async () => { const finding = createFinding({ location: undefined }); const result: TriggerResult = { @@ -306,7 +533,7 @@ describe('postTriggerReview', () => { }); vi.mocked(deduplicateFindings).mockResolvedValue({ newFindings: findings, - duplicateActions: [{ finding: findings[0]!, existingComment: createExistingComment(), matchType: 'hash', originalFindingId: findings[0]!.id }], + duplicateActions: [{ finding: findings[0]!, existingComment: createExistingComment(), matchType: 'hash' }], } as never); const postResult = await postTriggerReview({ @@ -349,7 +576,7 @@ describe('postTriggerReview', () => { vi.mocked(deduplicateFindings).mockResolvedValue({ newFindings: [findings[0]!], - duplicateActions: [{ finding: findings[1]!, existingComment: createExistingComment(), matchType: 'hash', originalFindingId: findings[1]!.id }], + duplicateActions: [{ finding: findings[1]!, existingComment: createExistingComment(), matchType: 'hash' }], } as never); // Dedup shrank the finding set, so the poster re-renders before posting. vi.mocked(renderSkillReport).mockReturnValue(createRenderResult({ @@ -377,8 +604,17 @@ describe('postTriggerReview', () => { // The gate verified twice: before duplicate processing and again before // the review write, where it saw the new head. expect(mockOctokit.pulls.get).toHaveBeenCalledTimes(2); - // No swallowed error: the findings were not marked failed. + // No swallowed error: the findings were not marked failed... expect(postResult.findingObservations.filter((o) => o.outcome === 'failed')).toEqual([]); + // ...but the candidate that would have posted isn't dropped either: it's + // recorded as skipped, so it isn't silently missing from the run's observations. + expect(postResult.findingObservations).toContainEqual({ + outcome: 'skipped', + finding: findings[0], + skill: 'test-skill', + skillExecutionId: undefined, + skippedReason: 'review_not_posted', + }); } finally { dateNowSpy.mockRestore(); } @@ -492,7 +728,7 @@ describe('postTriggerReview', () => { // Mock that the finding is a duplicate vi.mocked(deduplicateFindings).mockResolvedValue({ newFindings: [], - duplicateActions: [{ type: 'react_external', originalFindingId: finding.id, finding, existingComment, matchType: 'hash' }], + duplicateActions: [{ type: 'react_external', finding, existingComment, matchType: 'hash' }], }); vi.mocked(processDuplicateActions).mockResolvedValue({ updated: 0, reacted: 1, skipped: 0, failed: 0 }); @@ -511,7 +747,7 @@ describe('postTriggerReview', () => { }); expect(processDuplicateActions).toHaveBeenCalledWith( mockOctokit, 'test-owner', 'test-repo', - [{ type: 'react_external', originalFindingId: finding.id, finding, existingComment, matchType: 'hash' }], + [{ type: 'react_external', finding, existingComment, matchType: 'hash' }], 'test-skill' ); // Since all findings were duplicates and failOn not triggered, nothing new to post @@ -562,7 +798,7 @@ describe('postTriggerReview', () => { vi.mocked(deduplicateFindings).mockResolvedValue({ newFindings: [], - duplicateActions: [{ type: 'update_warden', originalFindingId: finding.id, finding, existingComment, matchType: 'hash' }], + duplicateActions: [{ type: 'update_warden', finding, existingComment, matchType: 'hash' }], }); vi.mocked(processDuplicateActions).mockResolvedValue({ updated: 0, reacted: 0, skipped: 1, failed: 0 }); @@ -626,7 +862,6 @@ describe('postTriggerReview', () => { }); const duplicateAction = { type: 'react_external' as const, - originalFindingId: duplicateFinding.id, finding: duplicateFinding, existingComment, matchType: 'hash' as const, @@ -691,7 +926,6 @@ describe('postTriggerReview', () => { duplicateActions: [ { type: 'update_warden', - originalFindingId: finding.id, finding, existingComment, matchType: 'hash', @@ -742,12 +976,12 @@ describe('postTriggerReview', () => { }; const existingComment = createExistingComment({ isWarden: true, findingId: 'WRZ-XPL' }); - const recenteredFinding = { ...finding, id: 'WRZ-XPL' }; + const recenteredFinding = { ...finding, reportedId: 'WRZ-XPL' }; // Mock that the finding is a duplicate (already posted in previous run) vi.mocked(deduplicateFindings).mockResolvedValue({ newFindings: [], - duplicateActions: [{ type: 'update_warden', originalFindingId: finding.id, finding: recenteredFinding, existingComment, matchType: 'hash' }], + duplicateActions: [{ type: 'update_warden', finding: recenteredFinding, existingComment, matchType: 'hash' }], }); vi.mocked(processDuplicateActions).mockResolvedValue({ updated: 1, reacted: 0, skipped: 0, failed: 0 }); @@ -776,13 +1010,14 @@ describe('postTriggerReview', () => { }); expect(processDuplicateActions).toHaveBeenCalledWith( mockOctokit, 'test-owner', 'test-repo', - [{ type: 'update_warden', originalFindingId: finding.id, finding: recenteredFinding, existingComment, matchType: 'hash' }], + [{ type: 'update_warden', finding: recenteredFinding, existingComment, matchType: 'hash' }], 'test-skill' ); // Even though all findings were deduplicated, REQUEST_CHANGES should still be posted expect(postResult.posted).toBe(true); expect([...postResult.activeWardenCommentIds]).toEqual([1]); - expect(result.report?.findings[0]?.id).toBe('WRZ-XPL'); + expect(result.report?.findings[0]?.id).toBe(finding.id); + expect(result.report?.findings[0]?.reportedId).toBe('WRZ-XPL'); expect(postResult.findingObservations).toEqual([ expect.objectContaining({ outcome: 'deduped', @@ -802,6 +1037,46 @@ describe('postTriggerReview', () => { ); }); + it('never touches findingProcessingEvents when dedupe sets reportedId, since id never mutates', async () => { + const finding = createFinding(); + const existingComment = createExistingComment({ isWarden: true, findingId: 'WRZ-XPL' }); + const recenteredFinding = { ...finding, reportedId: 'WRZ-XPL' }; + + const result: TriggerResult = { + triggerName: 'test-trigger', + skillName: 'test-skill', + report: { + skill: 'test-skill', + summary: 'Found 1 issue', + findings: [finding], + usage: { inputTokens: 100, outputTokens: 50, costUSD: 0.01 }, + }, + renderResult: createRenderResult(), + reportOn: 'low', + findingProcessingEvents: [ + { stage: 'verification', action: 'kept', finding, model: 'test-model' }, + ], + }; + + vi.mocked(deduplicateFindings).mockResolvedValue({ + newFindings: [], + duplicateActions: [{ type: 'react_external', finding: recenteredFinding, existingComment, matchType: 'hash' }], + }); + vi.mocked(processDuplicateActions).mockResolvedValue({ updated: 0, reacted: 1, skipped: 0, failed: 0 }); + + const ctx: ReviewPostingContext = { + result, + existingComments: [existingComment], + apiKey: 'test-key', + }; + + await postTriggerReview(ctx, mockDeps); + + expect(result.report?.findings[0]?.id).toBe(finding.id); + expect(result.report?.findings[0]?.reportedId).toBe('WRZ-XPL'); + expect(result.findingProcessingEvents?.[0]?.finding.id).toBe(finding.id); + }); + it('handles API errors gracefully', async () => { const finding = createFinding(); const result: TriggerResult = { diff --git a/packages/warden/src/action/review/poster.ts b/packages/warden/src/action/review/poster.ts index 4bd0e06c1..f0f6948ff 100644 --- a/packages/warden/src/action/review/poster.ts +++ b/packages/warden/src/action/review/poster.ts @@ -9,7 +9,7 @@ import type { Octokit } from '@octokit/rest'; import type { EventContext, Finding } from '../../types/index.js'; import { filterFindings } from '../../types/index.js'; import { shouldFail } from '../../triggers/matcher.js'; -import type { RenderResult } from '../../output/types.js'; +import type { GitHubComment, RenderResult } from '../../output/types.js'; import { renderSkillReport, renderFindingsBody } from '../../output/renderer.js'; import { deduplicateFindings, @@ -82,42 +82,51 @@ function emptyReviewPostResult( function buildDedupeObservations( actions: DeduplicateResult['duplicateActions'], - skill: string + skill: string, + skillExecutionId: string | undefined ): FindingObservation[] { return actions.map((action) => ({ outcome: 'deduped', finding: action.finding, skill, + skillExecutionId, dedupe: { source: action.existingComment.isWarden ? 'warden' : 'external', matchType: action.matchType, existingFindingId: action.existingComment.findingId, + existingSkillExecutionId: action.existingComment.skillExecutionId, ...(action.existingComment.id > 0 ? { existingCommentId: action.existingComment.id } : {}), existingThreadId: action.existingComment.threadId, existingResolved: action.existingComment.isResolved, + existingSkills: action.existingComment.skills, actor: action.existingComment.actor, }, })); } -function recenterReportFindingIds(reportFindings: Finding[], actions: DeduplicateResult['duplicateActions']): Finding[] { +/** + * `report.findings` holds the full pre-dedupe list; `duplicateActions` only + * covers the subset that matched an existing comment, as new `Finding` + * objects carrying `reportedId`. Sync that `reportedId` back onto the + * matching entries in `report.findings` by the (never-mutated) `id`, so + * every consumer of the full list sees the continuity id too. + */ +function syncReportedIds(reportFindings: Finding[], actions: DeduplicateResult['duplicateActions']): Finding[] { if (actions.length === 0) { return reportFindings; } - const ids = new Map( - actions - .filter((action) => action.originalFindingId !== action.finding.id) - .map((action) => [action.originalFindingId, action.finding.id]) + const reportedIds = new Map( + actions.filter((action) => action.finding.reportedId).map((action) => [action.finding.id, action.finding.reportedId]) ); - if (ids.size === 0) { + if (reportedIds.size === 0) { return reportFindings; } return reportFindings.map((finding) => { - const recenteredId = ids.get(finding.id); - return recenteredId ? { ...finding, id: recenteredId } : finding; + const reportedId = reportedIds.get(finding.id); + return reportedId ? { ...finding, reportedId } : finding; }); } @@ -134,6 +143,45 @@ function recenterReportFindingIds(reportFindings: Finding[], actions: Deduplicat */ type PostReviewOutcome = 'posted' | 'checks_only' | 'no_review' | 'blocked'; +/** A newly posted review comment's real GitHub identity, keyed by the finding it renders. */ +export type PostedCommentsByFindingId = Map; + +export interface PostReviewResult { + outcome: PostReviewOutcome; + commentsByFindingId?: PostedCommentsByFindingId; +} + +/** + * Match the review's own rendered comments (which carry `findingId`) against + * GitHub's response for the same review (which carries the real `id`/`html_url` + * but not `findingId`) by `path` + `line` + body text, since the API response + * doesn't echo back which request-side comment produced which result. + * Body text alone can collide when a batch renders two comments with + * identical text; pairing it with the comment's location narrows a match to + * comments GitHub itself would place at the same spot. + */ +function matchPostedCommentsToFindings( + rendered: GitHubComment[], + posted: { id: number; html_url: string; body: string; path?: string; line?: number | null }[] +): PostedCommentsByFindingId { + const byFindingId: PostedCommentsByFindingId = new Map(); + const availablePosted = [...posted]; + + for (const comment of rendered) { + if (!comment.findingId) continue; + const index = availablePosted.findIndex( + (p) => p.body === comment.body && p.path === comment.path && p.line === comment.line + ); + if (index === -1) continue; + const [match] = availablePosted.splice(index, 1); + if (match) { + byFindingId.set(comment.findingId, { id: match.id, url: match.html_url }); + } + } + + return byFindingId; +} + /** * Post a PR review to GitHub. */ @@ -142,13 +190,13 @@ async function postReviewToGitHub( context: EventContext, result: RenderResult, feedbackGate: ReviewFeedbackGate -): Promise { +): Promise { if (!context.pullRequest) { - return 'no_review'; + return { outcome: 'no_review' }; } if (!result.review) { - return 'no_review'; + return { outcome: 'no_review' }; } const { owner, name: repo } = context.repository; @@ -169,16 +217,16 @@ async function postReviewToGitHub( // Non-blocking body-only reviews cannot be resolved as review threads. // Keep those findings in Checks instead of leaving stale PR timeline entries. if (reviewComments.length === 0 && result.review.event === 'COMMENT') { - return 'checks_only'; + return { outcome: 'checks_only' }; } // Duplicate-action comment updates between the poster's gate check and this // write can outlive the gate's cache window; verify once more. if (!(await feedbackGate.canWrite())) { - return 'blocked'; + return { outcome: 'blocked' }; } - await octokit.pulls.createReview({ + const { data: review } = await octokit.pulls.createReview({ owner, repo, pull_number: pullNumber, @@ -188,7 +236,28 @@ async function postReviewToGitHub( comments: reviewComments, }); - return 'posted'; + if (reviewComments.length === 0) { + return { outcome: 'posted' }; + } + + try { + const postedComments = await octokit.paginate(octokit.pulls.listCommentsForReview, { + owner, + repo, + pull_number: pullNumber, + review_id: review.id, + per_page: 100, + }); + return { + outcome: 'posted', + commentsByFindingId: matchPostedCommentsToFindings(result.review.comments, postedComments), + }; + } catch (error) { + // The review itself posted successfully; losing the comment id/url lookup + // only degrades output-schema linkage, not the review the user sees. + warnAction(`Failed to fetch posted review comment ids: ${error}`); + return { outcome: 'posted' }; + } } /** @@ -251,6 +320,7 @@ export async function postTriggerReview( return emptyReviewPostResult(newComments, activeWardenCommentIds); } const skill = result.report.skill; + const skillExecutionId = result.skillExecutionId; // Filter findings by reportOn threshold and confidence const filteredFindings = filterFindings(result.report.findings, result.reportOn, result.minConfidence); @@ -295,6 +365,7 @@ export async function postTriggerReview( outcome: 'skipped', finding, skill, + skillExecutionId, skippedReason: 'duplicate_in_batch', }); } @@ -326,10 +397,10 @@ export async function postTriggerReview( currentSkill: skill, maxRetries: ctx.maxRetries, }); - result.report.findings = recenterReportFindingIds(result.report.findings, dedupResult.duplicateActions); + result.report.findings = syncReportedIds(result.report.findings, dedupResult.duplicateActions); findingsToPost = dedupResult.newFindings; findingsToMarkFailed = findingsToPost; - findingObservations.push(...buildDedupeObservations(dedupResult.duplicateActions, skill)); + findingObservations.push(...buildDedupeObservations(dedupResult.duplicateActions, skill, skillExecutionId)); // Merge dedup usage into the report's auxiliary usage if (dedupResult.dedupUsage) { @@ -427,13 +498,14 @@ export async function postTriggerReview( outcome: 'skipped', finding, skill, + skillExecutionId, skippedReason: 'max_findings', }); } - let postOutcome: PostReviewOutcome = 'no_review'; + let postResult: PostReviewResult = { outcome: 'no_review' }; try { - postOutcome = await postReviewToGitHub(octokit, context, renderResultToPost, deps.feedbackGate); + postResult = await postReviewToGitHub(octokit, context, renderResultToPost, deps.feedbackGate); } catch (error) { if (!isLineResolutionError(error)) { throw error; @@ -441,19 +513,25 @@ export async function postTriggerReview( if (renderResultToPost.review?.event === 'REQUEST_CHANGES') { warnAction(`Inline comments failed for ${result.triggerName}, posting findings in review body`); const fallback = moveCommentsToBody(renderResultToPost, postedFindings, skill); - postOutcome = await postReviewToGitHub(octokit, context, fallback, deps.feedbackGate); + postResult = await postReviewToGitHub(octokit, context, fallback, deps.feedbackGate); } else { warnAction(`Inline comments failed for ${result.triggerName}, falling back to checks only`); - postOutcome = 'checks_only'; + postResult = { outcome: 'checks_only' }; } } - if (postOutcome === 'checks_only') { + if (postResult.outcome === 'checks_only') { for (const finding of postedFindings) { - findingObservations.push({ outcome: 'skipped', finding, skill, skippedReason: 'no_inline_location' }); + findingObservations.push({ outcome: 'skipped', finding, skill, skillExecutionId, skippedReason: 'no_inline_location' }); } return emptyReviewPostResult(newComments, activeWardenCommentIds, findingObservations); } - if (postOutcome !== 'posted') { + if (postResult.outcome !== 'posted') { + // 'blocked' (a stale-head re-check) and 'no_review' both mean this pass + // never attempted to post — record the candidates as skipped rather than + // letting them vanish from the run's observations entirely. + for (const finding of postedFindings) { + findingObservations.push({ outcome: 'skipped', finding, skill, skillExecutionId, skippedReason: 'review_not_posted' }); + } return emptyReviewPostResult(newComments, activeWardenCommentIds, findingObservations); } // COMMENT reviews post with an empty body, so locationless findings that @@ -462,11 +540,19 @@ export async function postTriggerReview( const bodyStripped = renderResultToPost.review?.event === 'COMMENT'; for (const finding of postedFindings) { if (bodyStripped && !finding.location) { - findingObservations.push({ outcome: 'skipped', finding, skill, skippedReason: 'no_inline_location' }); + findingObservations.push({ outcome: 'skipped', finding, skill, skillExecutionId, skippedReason: 'no_inline_location' }); continue; } - findingObservations.push({ outcome: 'posted', finding, skill }); - const comment = findingToExistingComment(finding, skill); + const postedComment = postResult.commentsByFindingId?.get(finding.id); + findingObservations.push({ + outcome: 'posted', + finding, + skill, + skillExecutionId, + githubCommentId: postedComment?.id, + githubCommentUrl: postedComment?.url, + }); + const comment = findingToExistingComment(finding, skill, skillExecutionId); if (comment) { newComments.push(comment); } @@ -492,7 +578,7 @@ export async function postTriggerReview( activeWardenCommentIds, findingObservations: [ ...findingObservations, - ...findingsToMarkFailed.map((finding) => ({ outcome: 'failed' as const, finding, skill })), + ...findingsToMarkFailed.map((finding) => ({ outcome: 'failed' as const, finding, skill, skillExecutionId })), ], shouldFail: false, }; diff --git a/packages/warden/src/action/runner.test.ts b/packages/warden/src/action/runner.test.ts index 8c68fff5b..db6898178 100644 --- a/packages/warden/src/action/runner.test.ts +++ b/packages/warden/src/action/runner.test.ts @@ -49,6 +49,7 @@ const baseInputs: ActionInputs = { configPath: 'warden.toml', maxFindings: 50, parallel: 4, + outputSchemaVersion: '1', }; const capturedTransactions: TransactionEvent[] = []; diff --git a/packages/warden/src/action/triggers/executor.test.ts b/packages/warden/src/action/triggers/executor.test.ts index fd433f20e..c5b8d145e 100644 --- a/packages/warden/src/action/triggers/executor.test.ts +++ b/packages/warden/src/action/triggers/executor.test.ts @@ -87,6 +87,7 @@ describe('executeTrigger', () => { const check = await createSkillCheck(mockOctokit, skillName, checkOptions); return { url: check.url, + checkRunId: check.checkRunId, complete: (report: SkillReport, options: TriggerCheckCompleteOptions) => updateSkillCheck(mockOctokit, check.checkRunId, report, { ...checkOptions, @@ -118,6 +119,7 @@ describe('executeTrigger', () => { const mockTrigger: ResolvedTrigger = { id: 'test-trigger-id', + skillExecutionId: 'test-trigger-id', name: 'test-trigger', skill: 'test-skill', type: 'pull_request', @@ -246,6 +248,9 @@ describe('executeTrigger', () => { expect(result.report).toBe(mockReport); expect(result.renderResult).toBe(mockRenderResult); expect(result.error).toBeUndefined(); + expect(result.skillExecutionId).toBe('test-trigger-id'); + expect(result.checkRunId).toBe(123); + expect(result.checkRunUrl).toBe('https://github.com/check/123'); expect(createSkillCheck).toHaveBeenCalledWith(mockOctokit, 'test-skill', { owner: 'test-owner', repo: 'test-repo', @@ -286,6 +291,9 @@ describe('executeTrigger', () => { expect(result.triggerName).toBe('test-trigger'); expect(result.error).toBeDefined(); expect(result.report).toBeUndefined(); + expect(result.skillExecutionId).toBe('test-trigger-id'); + expect(result.checkRunId).toBe(123); + expect(result.checkRunUrl).toBe('https://github.com/check/123'); expect(failSkillCheck).toHaveBeenCalledWith( mockOctokit, 123, expect.objectContaining({ message: 'Skill not found' }), { owner: 'test-owner', repo: 'test-repo', headSha: 'abc123' } diff --git a/packages/warden/src/action/triggers/executor.ts b/packages/warden/src/action/triggers/executor.ts index 90affb87f..8204fb372 100644 --- a/packages/warden/src/action/triggers/executor.ts +++ b/packages/warden/src/action/triggers/executor.ts @@ -15,7 +15,8 @@ import type { OutputMode } from '../../cli/output/tty.js'; import { resolveSkillAsync } from '../../skills/loader.js'; import { filterContextByPaths } from '../../triggers/matcher.js'; import { runSkillTask, createDefaultCallbacks } from '../../cli/output/tasks.js'; -import type { SkillTaskOptions } from '../../cli/output/tasks.js'; +import type { SkillTaskOptions, SkillProgressCallbacks } from '../../cli/output/tasks.js'; +import type { FindingProcessingEvent } from '../../sdk/runner.js'; import { renderSkillReport } from '../../output/renderer.js'; import { logGroup, logGroupEnd } from '../workflow/base.js'; import { DEFAULT_FILE_CONCURRENCY, type AnalysisChunkingConfig } from '../../sdk/types.js'; @@ -58,6 +59,7 @@ function toAnalysisChunkingConfig( */ export interface TriggerCheckRun { url?: string; + checkRunId?: number; complete(report: SkillReport, options: TriggerCheckCompleteOptions): Promise; fail(error: unknown): Promise; } @@ -114,6 +116,7 @@ export interface TriggerResult { triggerId?: string; triggerName: string; skillName: string; + skillExecutionId?: string; report?: SkillReport; renderResult?: RenderResult; failOn?: SeverityThreshold; @@ -123,8 +126,13 @@ export interface TriggerResult { requestChanges?: boolean; failCheck?: boolean; checkRunUrl?: string; + checkRunId?: number; + issueNumber?: number; + issueUrl?: string; maxFindings?: number; error?: unknown; + /** Verification/merge events for this run, independent of CLI debug verbosity. */ + findingProcessingEvents?: FindingProcessingEvent[]; } // ----------------------------------------------------------------------------- @@ -155,10 +163,12 @@ export async function executeTrigger( // Create skill check (only for PRs) let skillCheck: TriggerCheckRun | undefined; let skillCheckUrl: string | undefined; + let skillCheckId: number | undefined; if (deps.checks && context.pullRequest) { try { skillCheck = await deps.checks.start(trigger.skill); skillCheckUrl = skillCheck.url; + skillCheckId = skillCheck.checkRunId; } catch (error) { console.error(`::warning::Failed to create skill check for ${trigger.skill}: ${error}`); } @@ -204,7 +214,15 @@ export async function executeTrigger( }, }; - const callbacks = createDefaultCallbacks([taskOptions], CI_OUTPUT_MODE, Verbosity.Normal); + const defaultCallbacks = createDefaultCallbacks([taskOptions], CI_OUTPUT_MODE, Verbosity.Normal); + const findingProcessingEvents: FindingProcessingEvent[] = []; + const callbacks: SkillProgressCallbacks = { + ...defaultCallbacks, + onFindingProcessing: (name, event) => { + findingProcessingEvents.push(event); + defaultCallbacks.onFindingProcessing?.(name, event); + }, + }; const fileConcurrency = deps.semaphore ? Number.MAX_SAFE_INTEGER : DEFAULT_FILE_CONCURRENCY; const result = await runSkillTask(taskOptions, fileConcurrency, callbacks, deps.semaphore); const report = result.report; @@ -259,6 +277,7 @@ export async function executeTrigger( triggerId: trigger.id, triggerName: trigger.name, skillName: trigger.skill, + skillExecutionId: trigger.skillExecutionId, report, renderResult, failOn, @@ -268,7 +287,9 @@ export async function executeTrigger( requestChanges, failCheck, checkRunUrl: skillCheckUrl, + checkRunId: skillCheckId, maxFindings, + findingProcessingEvents, }; } catch (error) { if (error instanceof ActionFailedError) throw error; @@ -288,7 +309,15 @@ export async function executeTrigger( console.error(`::warning::Trigger ${trigger.name} failed: ${error}`); logGroupEnd(); - return { triggerId: trigger.id, triggerName: trigger.name, skillName: trigger.skill, error }; + return { + triggerId: trigger.id, + triggerName: trigger.name, + skillName: trigger.skill, + skillExecutionId: trigger.skillExecutionId, + checkRunUrl: skillCheckUrl, + checkRunId: skillCheckId, + error, + }; } }, ); diff --git a/packages/warden/src/action/workflow/__fixtures__/schedule-with-pr-skill/warden.toml b/packages/warden/src/action/workflow/__fixtures__/schedule-with-pr-skill/warden.toml new file mode 100644 index 000000000..067429378 --- /dev/null +++ b/packages/warden/src/action/workflow/__fixtures__/schedule-with-pr-skill/warden.toml @@ -0,0 +1,16 @@ +version = 1 + +[[skills]] +name = "test-skill" +paths = ["src/**/*.ts"] + +[[skills.triggers]] +type = "schedule" + +[[skills]] +name = "pr-only-skill" +paths = ["src/**/*.ts"] + +[[skills.triggers]] +type = "pull_request" +actions = ["opened"] diff --git a/packages/warden/src/action/workflow/base.test.ts b/packages/warden/src/action/workflow/base.test.ts index 37c2a61e6..5b6752c1f 100644 --- a/packages/warden/src/action/workflow/base.test.ts +++ b/packages/warden/src/action/workflow/base.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import type * as NodeFS from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { EventContext, SkillReport } from '../../types/index.js'; @@ -14,13 +15,23 @@ vi.mock('../../utils/exec.js', async (importOriginal) => { }; }); +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, writeFileSync: vi.fn(actual.writeFileSync) }; +}); + import { execFileNonInteractive, execNonInteractive } from '../../utils/exec.js'; import { getFindingsOutputPath, + getFindingsOutputPathV2, prepareRuntimeEnvironment, writeFindingsOutput, + writeFindingsOutputs, + writeSchemaV2OutputPair, + writeSchemaV2OutputPairLive, } from './base.js'; import { FindingsOutputSchema } from '../reporting/output.js'; +import type { WardenFindingsV2, WardenMetadata } from '../reporting/output-v2.js'; const mockExecFile = vi.mocked(execFileNonInteractive); const mockExec = vi.mocked(execNonInteractive); @@ -99,6 +110,63 @@ describe('findings output', () => { expect(payload.findingObservations).toHaveLength(1); }); + it('repoints the findings-file output at the v2 file, matching what report mode reads as its findings-file input', () => { + process.env['GITHUB_WORKSPACE'] = tempDir; + + writeSchemaV2OutputPair(createV2Metadata(), createV2Findings(), createContext(tempDir)); + + const outputContent = readFileSync(process.env['GITHUB_OUTPUT']!, 'utf-8'); + expect(outputContent).toContain('metadata-file=warden-metadata.json\n'); + expect(outputContent).toContain('findings-file-v2=warden-findings-v2.json\n'); + expect(outputContent).toContain('findings-file=warden-findings-v2.json\n'); + }); + + it('writes a .done sidecar next to the findings file once both files land', () => { + process.env['GITHUB_WORKSPACE'] = tempDir; + + const { findingsPath } = writeSchemaV2OutputPair(createV2Metadata(), createV2Findings(), createContext(tempDir)); + + expect(existsSync(`${findingsPath}.done`)).toBe(true); + expect(readFileSync(`${findingsPath}.done`, 'utf-8')).toBe(''); + }); + + it('the live writer writes both content files but never a .done sidecar or GITHUB_OUTPUT', () => { + process.env['GITHUB_WORKSPACE'] = tempDir; + + writeSchemaV2OutputPairLive(createV2Metadata(), createV2Findings(), createContext(tempDir)); + + const findingsPath = getFindingsOutputPathV2(tempDir); + expect(existsSync(findingsPath)).toBe(true); + expect(existsSync(`${findingsPath}.done`)).toBe(false); + expect(existsSync(process.env['GITHUB_OUTPUT']!)).toBe(false); + }); + + it('the live writer removes a stale .done sidecar left over from a previous run', () => { + process.env['GITHUB_WORKSPACE'] = tempDir; + + // A prior run at this same path (persistent runner, or a repeated local + // invocation) finished and left its .done marker behind. + writeSchemaV2OutputPair(createV2Metadata(), createV2Findings(), createContext(tempDir)); + const findingsPath = getFindingsOutputPathV2(tempDir); + expect(existsSync(`${findingsPath}.done`)).toBe(true); + + writeSchemaV2OutputPairLive(createV2Metadata(), createV2Findings(), createContext(tempDir)); + + expect(existsSync(`${findingsPath}.done`)).toBe(false); + }); + + it('the live writer swallows a write failure instead of throwing', async () => { + process.env['GITHUB_WORKSPACE'] = tempDir; + const { writeFileSync } = await import('node:fs'); + vi.mocked(writeFileSync).mockImplementationOnce(() => { + throw new Error('disk full'); + }); + + expect(() => + writeSchemaV2OutputPairLive(createV2Metadata(), createV2Findings(), createContext(tempDir)) + ).not.toThrow(); + }); + it('falls back to RUNNER_TEMP when no repo path is provided', () => { const runnerTemp = join(tempDir, 'runner-temp'); mkdirSync(runnerTemp); @@ -108,6 +176,37 @@ describe('findings output', () => { }); }); +describe('writeFindingsOutputs', () => { + it('always runs writeV2 even when writeV1 throws, and reports the failure only after', () => { + const order: string[] = []; + const onFailure = vi.fn(); + + writeFindingsOutputs( + () => { + order.push('v1'); + throw new Error('disk full'); + }, + () => order.push('v2'), + onFailure + ); + + expect(order).toEqual(['v1', 'v2']); + expect(onFailure).toHaveBeenCalledWith('Failed to write findings output: Error: disk full'); + }); + + it('reports success and still runs writeV2 when writeV1 succeeds', () => { + const onFailure = vi.fn(); + const onSuccess = vi.fn(); + const writeV2 = vi.fn(); + + writeFindingsOutputs(() => '/tmp/warden-findings.json', writeV2, onFailure, onSuccess); + + expect(writeV2).toHaveBeenCalledOnce(); + expect(onSuccess).toHaveBeenCalledWith('/tmp/warden-findings.json'); + expect(onFailure).not.toHaveBeenCalled(); + }); +}); + describe('runtime setup', () => { let previousClaudeCodePath: string | undefined; let previousHome: string | undefined; @@ -175,10 +274,38 @@ function createInputs(overrides: Partial = {}): ActionInputs { configPath: 'warden.toml', maxFindings: 50, parallel: 4, + outputSchemaVersion: '1', ...overrides, }; } +function createV2Metadata(): WardenMetadata { + return { + schemaVersion: '2', + runId: '123', + generatedAt: new Date().toISOString(), + harness: { name: 'warden', version: '1.0.0' }, + repository: { owner: 'getsentry', name: 'warden', fullName: 'getsentry/warden' }, + event: 'pull_request', + }; +} + +function createV2Findings(): WardenFindingsV2 { + return { + schemaVersion: '2', + runId: '123', + skillExecutions: [], + findings: [], + findingObservations: [], + summary: { + totalFindings: 0, + totalSkillExecutions: 0, + bySeverity: { high: 0, medium: 0, low: 0 }, + byOutcome: { posted: 0, deduped: 0, skipped: 0, resolved: 0, failed: 0 }, + }, + }; +} + function createContext(repoPath: string): EventContext { return { eventType: 'schedule', diff --git a/packages/warden/src/action/workflow/base.ts b/packages/warden/src/action/workflow/base.ts index 45a24c78f..49ce2c4c0 100644 --- a/packages/warden/src/action/workflow/base.ts +++ b/packages/warden/src/action/workflow/base.ts @@ -4,21 +4,29 @@ * Shared infrastructure for PR and schedule workflows. */ -import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs'; +import { appendFileSync, existsSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join, relative } from 'node:path'; import { randomUUID } from 'node:crypto'; import type { Octokit } from '@octokit/rest'; import { execFileNonInteractive, execNonInteractive } from '../../utils/exec.js'; import { isRepoRelativePath, normalizePath } from '../../utils/path.js'; +import { writeFileAtomic, writeFilesAtomicPair } from '../../utils/fs.js'; import type { EventContext, SkillReport } from '../../types/index.js'; import type { FindingObservation } from '../reporting/outcomes.js'; import { buildFindingsOutput } from '../reporting/output.js'; import type { ReplayTriggerResult } from '../reporting/output.js'; +import { buildFindingsOutputV2, buildMetadataOutputV2 } from '../reporting/output-v2.js'; +import type { + BuildMetadataOutputV2Options, + WardenFindingsV2, + WardenMetadata, +} from '../reporting/output-v2.js'; import { countSeverity } from '../../triggers/matcher.js'; import type { RuntimeName } from '../../sdk/runtimes/index.js'; import type { ActionInputs } from '../inputs.js'; import type { TriggerResult } from '../triggers/executor.js'; +import type { ResolvedTrigger } from '../../config/loader.js'; /** * Sentinel error thrown by setFailed() so the top-level catch handler @@ -380,11 +388,15 @@ export function writeFindingsOutput( reports: SkillReport[], context: EventContext, findingObservations: FindingObservation[] = [], - options: { triggerResults?: ReplayTriggerResult[] } = {} + options: { + triggerResults?: ReplayTriggerResult[]; + configuredSkills?: { name: string; triggered: boolean }[]; + } = {} ): string { const filePath = getFindingsOutputPath(context.repoPath); const output = buildFindingsOutput(reports, context, findingObservations, { triggerResults: options.triggerResults, + configuredSkills: options.configuredSkills, }); mkdirSync(dirname(filePath), { recursive: true }); @@ -392,3 +404,165 @@ export function writeFindingsOutput( setOutput('findings-file', getFindingsOutputValue(filePath, context.repoPath)); return filePath; } + +/** + * Write the v1 findings file, then always run `writeV2` — even if the v1 + * write throws — since v2 consumers must never see a missing pair when v1 + * output was attempted. A v1 failure is reported through `onFailure` only + * after `writeV2` has run, so callers that need a hard failure (e.g. analyze + * mode, whose output feeds report mode) can pass a throwing handler without + * risking the v2 write becoming unreachable. + */ +export function writeFindingsOutputs( + writeV1: () => string, + writeV2: () => void, + onFailure: (message: string) => void, + onSuccess?: (path: string) => void +): void { + let failureMessage: string | undefined; + try { + const path = writeV1(); + onSuccess?.(path); + } catch (error) { + failureMessage = `Failed to write findings output: ${error}`; + } + writeV2(); + if (failureMessage !== undefined) { + onFailure(failureMessage); + } +} + +/** Get the path for the schema-v2 metadata output file. */ +export function getMetadataOutputPath(repoPath?: string): string { + if (repoPath && process.env['GITHUB_WORKSPACE']) { + return join(repoPath, 'warden-metadata.json'); + } + + const tmpDir = process.env['RUNNER_TEMP'] ?? tmpdir(); + return join(tmpDir, 'warden-metadata.json'); +} + +/** Get the path for the schema-v2 findings output file. */ +export function getFindingsOutputPathV2(repoPath?: string): string { + if (repoPath && process.env['GITHUB_WORKSPACE']) { + return join(repoPath, 'warden-findings-v2.json'); + } + + const tmpDir = process.env['RUNNER_TEMP'] ?? tmpdir(); + return join(tmpDir, 'warden-findings-v2.json'); +} + +/** + * Write both schema-v2 files atomically, then set their action outputs + * together, then mark the pair done with a sidecar file next to the + * findings file. + * + * `metadata-file` and `findings-file` must never disagree on which schema + * version they point at. `writeFilesAtomicPair` stages both files' content + * before renaming either into place, so a staging failure on one file never + * leaves the other committed to a new run while its partner still holds a + * prior one. Setting outputs only after that call means a failure here + * leaves neither v2 output set, so a prior v1 `findings-file` output remains + * the last consistent value. + * + * This is the run's one true "done" checkpoint — every intermediate write + * during the run goes through `writeSchemaV2OutputPairLive` instead, which + * never touches the `.done` sidecar or these action outputs. + */ +export function writeSchemaV2OutputPair( + metadata: WardenMetadata, + findings: WardenFindingsV2, + context: EventContext +): { metadataPath: string; findingsPath: string } { + const metadataPath = getMetadataOutputPath(context.repoPath); + const findingsPath = getFindingsOutputPathV2(context.repoPath); + + writeFilesAtomicPair([ + { path: metadataPath, content: JSON.stringify(metadata, null, 2) }, + { path: findingsPath, content: JSON.stringify(findings, null, 2) }, + ]); + writeFileAtomic(`${findingsPath}.done`, ''); + + setOutput('metadata-file', getFindingsOutputValue(metadataPath, context.repoPath)); + const findingsOutputValue = getFindingsOutputValue(findingsPath, context.repoPath); + setOutput('findings-file-v2', findingsOutputValue); + setOutput('findings-file', findingsOutputValue); + + return { metadataPath, findingsPath }; +} + +/** + * Write both schema-v2 files atomically, without the `.done` sidecar or any + * action outputs. Used for the in-progress writes fired as each trigger + * completes during a run; the real, single `.done`-marked write still + * happens once via `writeSchemaV2OutputPair` after the run finishes. Never + * throws — a transient write hiccup here must not abort a run the way a + * final-write failure legitimately can. + * + * Removes any `.done` sidecar left over from a previous run at this same + * path before writing. On an ephemeral GitHub-hosted runner this never + * exists, but a persistent/self-hosted runner or a repeated local `warden + * runs follow` invocation reusing the same paths could otherwise see a + * stale `.done` and report a brand-new, still in-progress run as finished. + */ +export function writeSchemaV2OutputPairLive( + metadata: WardenMetadata, + findings: WardenFindingsV2, + context: EventContext +): void { + try { + const metadataPath = getMetadataOutputPath(context.repoPath); + const findingsPath = getFindingsOutputPathV2(context.repoPath); + if (existsSync(`${findingsPath}.done`)) { + try { unlinkSync(`${findingsPath}.done`); } catch { /* best-effort cleanup */ } + } + writeFilesAtomicPair([ + { path: metadataPath, content: JSON.stringify(metadata, null, 2) }, + { path: findingsPath, content: JSON.stringify(findings, null, 2) }, + ]); + } catch (error) { + console.error(`::warning::Failed to write live schema-v2 output: ${error}`); + } +} + +/** Build and write the schema-v2 metadata/findings pair from raw trigger results. */ +export function writeSchemaV2Output( + context: EventContext, + resolvedTriggers: ResolvedTrigger[], + matchedTriggers: ResolvedTrigger[], + results: TriggerResult[], + findingObservations: FindingObservation[], + options: BuildMetadataOutputV2Options +): { metadataPath: string; findingsPath: string } { + const metadata = buildMetadataOutputV2(context, resolvedTriggers, matchedTriggers, results, options); + const findings = buildFindingsOutputV2(results, matchedTriggers, findingObservations, { runId: options.runId }); + return writeSchemaV2OutputPair(metadata, findings, context); +} + +/** Build and write the in-progress schema-v2 metadata/findings pair from partial trigger results. */ +export function writeSchemaV2OutputLive( + context: EventContext, + resolvedTriggers: ResolvedTrigger[], + matchedTriggers: ResolvedTrigger[], + results: TriggerResult[], + findingObservations: FindingObservation[], + options: BuildMetadataOutputV2Options +): void { + const metadata = buildMetadataOutputV2(context, resolvedTriggers, matchedTriggers, results, options); + const findings = buildFindingsOutputV2(results, matchedTriggers, findingObservations, { runId: options.runId }); + writeSchemaV2OutputPairLive(metadata, findings, context); +} + +/** Shared `{runId, runAttempt, actionRef, ...}` options every schema-v2 write site builds from `ActionInputs`. */ +export function buildV2WriteOptions(inputs: ActionInputs): BuildMetadataOutputV2Options { + return { + runId: process.env['GITHUB_RUN_ID'] ?? '', + runAttempt: process.env['GITHUB_RUN_ATTEMPT'], + actionRef: inputs.actionRef, + failOn: inputs.failOn, + reportOn: inputs.reportOn, + failCheck: inputs.failCheck, + requestChanges: inputs.requestChanges, + maxFindings: inputs.maxFindings, + }; +} diff --git a/packages/warden/src/action/workflow/pr-workflow.test.ts b/packages/warden/src/action/workflow/pr-workflow.test.ts index 52a988ea6..103c03730 100644 --- a/packages/warden/src/action/workflow/pr-workflow.test.ts +++ b/packages/warden/src/action/workflow/pr-workflow.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -26,6 +26,7 @@ const RUNTIME_CLAUDE_FIXTURES_DIR = join(FIXTURES_DIR, 'runtime-claude'); const EMPTY_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'empty-auxiliary-model'); const LAYERED_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'layered-auxiliary-model'); const NO_MATCH_EMPTY_AUXILIARY_MODEL_FIXTURES_DIR = join(FIXTURES_DIR, 'no-match-empty-auxiliary-model'); +const SCHEDULE_ONLY_FIXTURES_DIR = join(FIXTURES_DIR, 'schedule'); const EVENT_PAYLOAD_PATH = join(FIXTURES_DIR, 'event-payloads/pull_request_opened.json'); const PR_HEAD_SHA = 'abc123def456'; const PREVIOUS_HEAD_SHA = 'previous123sha456'; @@ -118,7 +119,7 @@ vi.mock('./base.js', async () => { import { runSkillTask } from '../../cli/output/tasks.js'; import { fetchExistingComments, deduplicateFindings, processDuplicateActions } from '../../output/dedup.js'; import { evaluateFixAttempts } from '../fix-evaluation/index.js'; -import { setFailed, writeFindingsOutput } from './base.js'; +import { setFailed, writeFindingsOutput, getMetadataOutputPath, getFindingsOutputPathV2 } from './base.js'; import { runPRWorkflow } from './pr-workflow.js'; import { clearSkillsCache } from '../../skills/loader.js'; import { Semaphore } from '../../utils/index.js'; @@ -228,6 +229,7 @@ function createDefaultInputs(overrides: Partial = {}): ActionInput configPath: 'warden.toml', maxFindings: 50, parallel: 2, + outputSchemaVersion: '1', ...overrides, }; } @@ -381,10 +383,58 @@ describe('runPRWorkflow', () => { report, }), ], + configuredSkills: [{ name: 'test-skill', triggered: true }], } ); }); + it('analyze mode lists a schedule-only skill as configured but not triggered on a PR run', async () => { + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + SCHEDULE_ONLY_FIXTURES_DIR + ); + + expect(mockRunSkillTask).not.toHaveBeenCalled(); + expect(mockWriteFindingsOutput).toHaveBeenCalledWith( + [], + expect.objectContaining({ + repository: expect.objectContaining({ fullName: 'test-owner/test-repo' }), + }), + [], + { + triggerResults: [], + configuredSkills: [{ name: 'test-skill', triggered: false }], + } + ); + }); + + it('analyze mode writes schema-v2 outputs even when no triggers matched', async () => { + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze', outputSchemaVersion: '2' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + SCHEDULE_ONLY_FIXTURES_DIR + ); + + const metadataFile = getMetadataOutputPath(SCHEDULE_ONLY_FIXTURES_DIR); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_ONLY_FIXTURES_DIR); + try { + const metadata = JSON.parse(readFileSync(metadataFile, 'utf-8')); + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + expect(metadata.configuredSkills).toEqual([{ name: 'test-skill', triggered: false }]); + expect(findings.skillExecutions).toEqual([]); + expect(findings.findings).toEqual([]); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + it('analyze mode fails when the findings artifact cannot be written', async () => { const report = createSkillReport({ findings: [createFinding()] }); mockRunSkillTask.mockResolvedValue({ name: 'test-trigger', report }); @@ -409,6 +459,40 @@ describe('runPRWorkflow', () => { expect(mockOctokit.pulls.createReview).not.toHaveBeenCalled(); }); + it('analyze mode still writes schema-v2 outputs when the v1 findings artifact fails to write', async () => { + const report = createSkillReport({ findings: [createFinding()] }); + mockRunSkillTask.mockResolvedValue({ name: 'test-trigger', report }); + mockWriteFindingsOutput.mockImplementationOnce(() => { + throw new Error('Disk full'); + }); + + const metadataFile = getMetadataOutputPath(FIXTURES_DIR); + const findingsFile = getFindingsOutputPathV2(FIXTURES_DIR); + try { + await expect( + runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze', outputSchemaVersion: '2' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + FIXTURES_DIR + ) + ).rejects.toThrow('setFailed'); + + expect(mockSetFailed).toHaveBeenCalledWith( + expect.stringContaining('Failed to write findings output: Error: Disk full') + ); + const metadata = JSON.parse(readFileSync(metadataFile, 'utf-8')); + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + expect(metadata.configuredSkills).toEqual([{ name: 'test-skill', triggered: true }]); + expect(findings.findings).toHaveLength(1); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + it('report mode publishes completed checks from the findings file without rerunning skills', async () => { const finding = createFinding(); const report = createSkillReport({ findings: [finding] }); @@ -456,6 +540,240 @@ describe('runPRWorkflow', () => { ); }); + it('schema v2 analyze output replays correctly in report mode', async () => { + const finding = createFinding(); + const report = createSkillReport({ findings: [finding] }); + mockRunSkillTask.mockResolvedValue({ name: 'test-trigger', report }); + + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze', outputSchemaVersion: '2' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + FIXTURES_DIR + ); + + const metadataFile = getMetadataOutputPath(FIXTURES_DIR); + const findingsFile = getFindingsOutputPathV2(FIXTURES_DIR); + + let findingsFileAfterReport: string; + try { + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'report', outputSchemaVersion: '2', metadataFile, findingsFile }), + 'pull_request', + EVENT_PAYLOAD_PATH, + FIXTURES_DIR + ); + findingsFileAfterReport = readFileSync(findingsFile, 'utf-8'); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + + expect(mockRunSkillTask).toHaveBeenCalledTimes(1); + expect(mockOctokit.checks.create).toHaveBeenCalledWith( + expect.objectContaining({ name: 'warden: test-skill', status: 'completed' }) + ); + expect(mockOctokit.pulls.createReview).toHaveBeenCalledWith( + expect.objectContaining({ + owner: 'test-owner', + repo: 'test-repo', + pull_number: 123, + commit_id: PR_HEAD_SHA, + }) + ); + + // Regression: report mode must rewrite the v2 findings file with the + // real posting outcome, not leave analyze mode's empty findingObservations. + const findingsV2 = JSON.parse(findingsFileAfterReport); + expect(findingsV2.findingObservations).toContainEqual( + expect.objectContaining({ outcome: 'posted' }) + ); + }); + + it('report mode does not cross-attribute findings that share an id across duplicate trigger executions', async () => { + let invocation = 0; + mockRunSkillTask.mockImplementation(async (taskOptions) => { + invocation++; + return { + name: taskOptions.name, + report: createSkillReport({ + skill: 'test-skill', + findings: [ + createFinding({ + id: 'collide', + title: `Finding from execution ${invocation}`, + location: { path: 'src/test.ts', startLine: invocation }, + }), + ], + }), + }; + }); + + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze', outputSchemaVersion: '2' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + DUPLICATE_TRIGGER_FIXTURES_DIR + ); + + const metadataFile = getMetadataOutputPath(DUPLICATE_TRIGGER_FIXTURES_DIR); + const findingsFile = getFindingsOutputPathV2(DUPLICATE_TRIGGER_FIXTURES_DIR); + + try { + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'report', outputSchemaVersion: '2', metadataFile, findingsFile }), + 'pull_request', + EVENT_PAYLOAD_PATH, + DUPLICATE_TRIGGER_FIXTURES_DIR + ); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + + const postedBodies = vi.mocked(mockOctokit.pulls.createReview).mock.calls + .flatMap((call) => call[0]?.comments ?? []) + .map((comment) => comment.body); + + expect(postedBodies.some((body) => body.includes('Finding from execution 1'))).toBe(true); + expect(postedBodies.some((body) => body.includes('Finding from execution 2'))).toBe(true); + }); + + it('report mode rejects a v2 findings file with ambiguous duplicate execution triggerIds', async () => { + mockRunSkillTask.mockResolvedValue({ name: 'test-trigger', report: createSkillReport() }); + + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze', outputSchemaVersion: '2' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + FIXTURES_DIR + ); + + const metadataFile = getMetadataOutputPath(FIXTURES_DIR); + const findingsFile = getFindingsOutputPathV2(FIXTURES_DIR); + + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + findings.skillExecutions.push({ ...findings.skillExecutions[0], skillExecutionId: 'exec-duplicate' }); + writeFileSync(findingsFile, JSON.stringify(findings, null, 2)); + + try { + await expect( + runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'report', outputSchemaVersion: '2', metadataFile, findingsFile }), + 'pull_request', + EVENT_PAYLOAD_PATH, + FIXTURES_DIR + ) + ).rejects.toThrow('ambiguous duplicate trigger result'); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('report mode rejects a v2 findings file with executions that no longer match current config', async () => { + mockRunSkillTask.mockResolvedValue({ name: 'test-trigger', report: createSkillReport() }); + + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze', outputSchemaVersion: '2' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + FIXTURES_DIR + ); + + const metadataFile = getMetadataOutputPath(FIXTURES_DIR); + const findingsFile = getFindingsOutputPathV2(FIXTURES_DIR); + + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + findings.skillExecutions.push({ + ...findings.skillExecutions[0], + triggerId: 'stale-trigger-removed-from-config', + skillExecutionId: 'exec-stale', + skillName: 'removed-skill', + }); + writeFileSync(findingsFile, JSON.stringify(findings, null, 2)); + + try { + await expect( + runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'report', outputSchemaVersion: '2', metadataFile, findingsFile }), + 'pull_request', + EVENT_PAYLOAD_PATH, + FIXTURES_DIR + ) + ).rejects.toThrow('do not match current config'); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('does not guess a skillExecutionId for a resolved comment when multiple triggers share its skill name', async () => { + mockRunSkillTask.mockResolvedValue({ name: 'test-trigger', report: createSkillReport() }); + + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze', outputSchemaVersion: '2' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + DUPLICATE_TRIGGER_FIXTURES_DIR + ); + + const metadataFile = getMetadataOutputPath(DUPLICATE_TRIGGER_FIXTURES_DIR); + const findingsFile = getFindingsOutputPathV2(DUPLICATE_TRIGGER_FIXTURES_DIR); + + mockFetchExistingComments.mockResolvedValue([ + createExistingWardenComment({ skills: ['test-skill'] }), + ]); + mockEvaluateFixAttempts.mockResolvedValue({ + toResolve: [createExistingWardenComment({ skills: ['test-skill'] })], + toReply: [], + evaluations: [], + skipped: 0, + evaluated: 1, + failedEvaluations: 0, + uniqueFindingsEvaluated: 1, + uniqueFindingsCodeChanged: 1, + uniqueFindingsResolved: 1, + usage: { inputTokens: 0, outputTokens: 0, costUSD: 0 }, + }); + + let findingsFileAfterReport: string; + try { + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'report', outputSchemaVersion: '2', metadataFile, findingsFile }), + 'pull_request', + EVENT_PAYLOAD_PATH, + DUPLICATE_TRIGGER_FIXTURES_DIR + ); + findingsFileAfterReport = readFileSync(findingsFile, 'utf-8'); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + + const findingsV2 = JSON.parse(findingsFileAfterReport); + const resolved = findingsV2.findingObservations.find( + (o: { outcome: string }) => o.outcome === 'resolved' + ); + expect(resolved?.origin.skillName).toBe('test-skill'); + expect(resolved?.origin.skillExecutionId).toBeFalsy(); + }); + it('report mode renders checks and reviews from report-step inputs', async () => { const report = createSkillReport({ findings: [ @@ -1084,7 +1402,6 @@ describe('runPRWorkflow', () => { duplicateActions: [ { type: 'update_warden', - originalFindingId: finding.id, finding, existingComment, matchType: 'hash', @@ -1212,6 +1529,49 @@ describe('runPRWorkflow', () => { } }); + it('records a skipped observation, not a silently missing one, when a trigger\'s review never gets attempted due to a stale head', async () => { + // Same race as 'stops review feedback if the PR head advances before + // posting' above, but checked from the schema-v2 output side: the + // finding must not simply vanish from findingObservations. + let now = 1_750_000_000_000; + const dateNowSpy = vi.spyOn(Date, 'now').mockImplementation(() => now); + try { + const finding = createFinding(); + const report = createSkillReport({ findings: [finding] }); + vi.mocked(mockOctokit.pulls.get) + .mockResolvedValueOnce(createGetPullResponse(PR_HEAD_SHA)) + .mockResolvedValueOnce(createGetPullResponse('new-head-sha')); + + mockFetchExistingComments.mockImplementation(async () => { + now += 60_000; + return [createExistingWardenComment()]; + }); + mockRunSkillTask.mockResolvedValue({ name: 'test-trigger', report }); + + await runPRWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + 'pull_request', EVENT_PAYLOAD_PATH, FIXTURES_DIR + ); + + expect(mockOctokit.pulls.createReview).not.toHaveBeenCalled(); + + const findingsFile = getFindingsOutputPathV2(FIXTURES_DIR); + try { + const findingsV2 = JSON.parse(readFileSync(findingsFile, 'utf-8')); + expect(findingsV2.findingObservations).toContainEqual( + expect.objectContaining({ outcome: 'skipped', skippedReason: 'review_not_posted' }) + ); + } finally { + rmSync(getMetadataOutputPath(FIXTURES_DIR), { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + } finally { + dateNowSpy.mockRestore(); + } + }); + it('stops stale resolution and dismissal if the PR head advances after posting', async () => { // Advance the clock past the gate TTL inside the review post so the // resolve phase re-verifies the head and sees it advanced. @@ -1336,7 +1696,6 @@ describe('runPRWorkflow', () => { duplicateActions: [ { type: 'react_external', - originalFindingId: finding.id, finding, existingComment: { id: 1, @@ -1474,6 +1833,54 @@ describe('runPRWorkflow', () => { expect(maxActiveRuns).toBe(1); }); + it('writes live schema-v2 output after each trigger completes, before the run finishes', async () => { + let invocation = 0; + let releaseSecond!: () => void; + const secondGate = new Promise((resolve) => { + releaseSecond = resolve; + }); + + mockRunSkillTask.mockImplementation(async (taskOptions) => { + invocation++; + if (invocation === 2) await secondGate; + return { + name: taskOptions.name, + report: createSkillReport({ skill: 'test-skill' }), + }; + }); + + const findingsFile = getFindingsOutputPathV2(DUPLICATE_TRIGGER_FIXTURES_DIR); + const metadataFile = getMetadataOutputPath(DUPLICATE_TRIGGER_FIXTURES_DIR); + + const workflow = runPRWorkflow( + mockOctokit, + createDefaultInputs({ mode: 'analyze', outputSchemaVersion: '2' }), + 'pull_request', + EVENT_PAYLOAD_PATH, + DUPLICATE_TRIGGER_FIXTURES_DIR + ); + + try { + await vi.waitFor(() => { + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + expect(findings.skillExecutions).toHaveLength(1); + }); + + expect(existsSync(`${findingsFile}.done`)).toBe(false); + + releaseSecond(); + await workflow; + + const finalFindings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + expect(finalFindings.skillExecutions).toHaveLength(2); + expect(existsSync(`${findingsFile}.done`)).toBe(true); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + it('records trigger failure and updates check before failing', async () => { // When all triggers fail, the workflow should still update the check // before calling setFailed. @@ -2134,7 +2541,6 @@ describe('runPRWorkflow', () => { duplicateActions: [ { type: 'update_warden', - originalFindingId: finding.id, finding, existingComment, matchType: 'semantic', @@ -2410,7 +2816,10 @@ describe('runPRWorkflow', () => { skill: undefined, resolvedReason: 'fix_evaluation', }), - ] + ], + { + configuredSkills: [{ name: 'test-skill', triggered: false }], + } ); }); diff --git a/packages/warden/src/action/workflow/pr-workflow.ts b/packages/warden/src/action/workflow/pr-workflow.ts index 879668db2..f580b25d5 100644 --- a/packages/warden/src/action/workflow/pr-workflow.ts +++ b/packages/warden/src/action/workflow/pr-workflow.ts @@ -77,13 +77,29 @@ import { setWorkflowOutputs, getAuthenticatedBotLogin, writeFindingsOutput, + writeFindingsOutputs, + writeSchemaV2Output, + writeSchemaV2OutputLive, + writeSchemaV2OutputPair, + buildV2WriteOptions, } from './base.js'; import { renderSkillReport } from '../../output/renderer.js'; import { FindingsOutputSchema, + buildConfiguredSkillsList, type FindingsOutput, type ReplayTriggerResult, } from '../reporting/output.js'; +import { + WardenMetadataSchema, + WardenFindingsSchemaV2, + patchFindingsOutputV2Observations, + fromAuxiliaryUsageEntries, + skillExecutionIdByNameFrom, + toFindingFromV2, + type WardenMetadata, + type WardenFindingsV2, +} from '../reporting/output-v2.js'; // ----------------------------------------------------------------------------- // Phase Result Types @@ -93,6 +109,7 @@ interface InitResult { context: EventContext; runnerConcurrency?: number; auxiliaryOptions: AuxiliaryWorkflowOptions; + resolvedTriggers: ResolvedTrigger[]; matchedTriggers: ResolvedTrigger[]; skippedTriggers: ResolvedTrigger[]; skipCoreCheck?: SkippedCoreCheck; @@ -345,7 +362,14 @@ async function initializeWorkflow( console.log('No triggers matched for this event'); } - return { context, runnerConcurrency, auxiliaryOptions, matchedTriggers, skippedTriggers }; + return { + context, + runnerConcurrency, + auxiliaryOptions, + resolvedTriggers, + matchedTriggers, + skippedTriggers, + }; } catch (error) { if ( error instanceof ConfigLoadError && @@ -358,6 +382,7 @@ async function initializeWorkflow( context, runnerConcurrency, auxiliaryOptions, + resolvedTriggers: [], matchedTriggers: [], skippedTriggers: [], skipCoreCheck: { @@ -463,6 +488,7 @@ function createTriggerCheckReporter( const check = await createSkillCheck(octokit, skillName, checkOptions); return { url: check.url, + checkRunId: check.checkRunId, complete: (report, options) => updateSkillCheck(octokit, check.checkRunId, report, { ...checkOptions, @@ -479,7 +505,7 @@ async function executeAllTriggers( context: EventContext, runnerConcurrency: number | undefined, inputs: ActionInputs, - options: { checks?: TriggerCheckReporter } = {} + options: { checks?: TriggerCheckReporter; onTriggerComplete?: (result: TriggerResult) => void } = {} ): Promise { const concurrency = runnerConcurrency ?? inputs.parallel; const runtimeEnv = await prepareRuntimeEnvironment(matchedTriggers, inputs); @@ -506,6 +532,9 @@ async function executeAllTriggers( abortController, circuitBreaker, checks: options.checks, + }).then((result) => { + try { options.onTriggerComplete?.(result); } catch { /* live output must never abort the pool */ } + return result; }), { shouldAbort: () => abortController.signal.aborted }, ); @@ -588,6 +617,20 @@ async function postReviewsAndTrackFailures( postResult.activeWardenCommentIds.forEach((id) => activeWardenCommentIds.add(id)); findingObservations.push(...postResult.findingObservations); reviewPosted = postResult.posted; + } else { + // The gate never let this trigger's review post at all — record its + // reportable findings as skipped rather than leaving them with zero + // observations, mirroring postTriggerReview's own 'blocked'/'no_review' handling. + const skippedFindings = filterFindings(result.report.findings, result.reportOn, result.minConfidence); + for (const finding of skippedFindings) { + findingObservations.push({ + outcome: 'skipped', + finding, + skill: result.report.skill, + skillExecutionId: result.skillExecutionId, + skippedReason: 'review_not_posted', + }); + } } // A stale head skips silently (the newer run owns feedback), but an // unverifiable head must not silently swallow a blocking review. @@ -656,6 +699,7 @@ async function evaluateFixesAndResolveStale( anthropicApiKey: string, auxiliaryOptions: AuxiliaryWorkflowOptions, gate: ReviewFeedbackGate, + matchedTriggers: ResolvedTrigger[], options: { failOnWriteError?: boolean } = {} ): Promise<{ allResolved: boolean; @@ -663,6 +707,7 @@ async function evaluateFixesAndResolveStale( autoResolvedByStaleCheck: number; findingObservations: FindingObservation[]; }> { + const skillExecutionIdByName = skillExecutionIdByNameFrom(matchedTriggers); const wardenComments = fetchedComments.filter((c) => c.isWarden); const commentsResolvedByFixEval = new Set(); const commentsEvaluatedByFixEval = new Set(); @@ -786,6 +831,7 @@ async function evaluateFixesAndResolveStale( outcome: 'resolved', finding: existingCommentToFinding(comment), skill: comment.skills?.[0], + skillExecutionId: skillExecutionIdByName.get(comment.skills?.[0] ?? ''), resolvedReason: 'fix_evaluation', }); } @@ -886,6 +932,7 @@ async function evaluateFixesAndResolveStale( outcome: 'resolved', finding: existingCommentToFinding(comment), skill: comment.skills?.[0], + skillExecutionId: skillExecutionIdByName.get(comment.skills?.[0] ?? ''), resolvedReason: 'stale_check', }); } @@ -965,6 +1012,75 @@ async function dismissPreviousReviewIfResolved( } } +/** + * Write the schema-v2 metadata/findings pair when opted in. Called from every + * v1 findings-file write site, including early-return "no triggers matched" + * paths, so v2 consumers never see a missing pair when v1 output exists. + */ +function writeSchemaV2Outputs( + inputs: ActionInputs, + context: EventContext, + resolvedTriggers: ResolvedTrigger[], + matchedTriggers: ResolvedTrigger[], + results: TriggerResult[], + findingObservations: FindingObservation[], + onError: (message: string) => void +): void { + if (inputs.outputSchemaVersion !== '2') return; + + try { + const { metadataPath, findingsPath } = writeSchemaV2Output( + context, resolvedTriggers, matchedTriggers, results, findingObservations, + buildV2WriteOptions(inputs) + ); + logAction(`Metadata written to ${metadataPath}`); + logAction(`Findings (v2) written to ${findingsPath}`); + } catch (error) { + onError(`Failed to write schema-v2 output: ${error}`); + } +} + +/** Live counterpart of {@link writeSchemaV2Outputs} — fired after each trigger completes, never fatal. */ +function writeSchemaV2OutputsLive( + inputs: ActionInputs, + context: EventContext, + resolvedTriggers: ResolvedTrigger[], + matchedTriggers: ResolvedTrigger[], + results: TriggerResult[] +): void { + if (inputs.outputSchemaVersion !== '2') return; + writeSchemaV2OutputLive(context, resolvedTriggers, matchedTriggers, results, [], buildV2WriteOptions(inputs)); +} + +/** + * Report mode's v2 write: unlike analyze mode or single-run mode, report mode + * only has TriggerResults replayed from ExportedFindingV2 (no + * findingProcessingEvents), so a full rebuild here would silently wipe the + * analyze-phase `provenance`/`discardedFindings`. Instead, write the + * unmodified analyze-phase metadata and patch only `findingObservations` / + * `summary.byOutcome` onto the analyze-phase findings payload. + */ +function writeSchemaV2ReportOutputs( + metadataOutputV2: WardenMetadata | undefined, + findingsOutputV2: WardenFindingsV2 | undefined, + context: EventContext, + results: TriggerResult[], + matchedTriggers: ResolvedTrigger[], + findingObservations: FindingObservation[], + onError: (message: string) => void +): void { + if (!metadataOutputV2 || !findingsOutputV2) return; + + try { + const patched = patchFindingsOutputV2Observations(findingsOutputV2, results, matchedTriggers, findingObservations); + const { metadataPath, findingsPath } = writeSchemaV2OutputPair(metadataOutputV2, patched, context); + logAction(`Metadata written to ${metadataPath}`); + logAction(`Findings (v2) written to ${findingsPath}`); + } catch (error) { + onError(`Failed to write schema-v2 output: ${error}`); + } +} + /** * Dismiss review, set outputs, update core check, fail action. */ @@ -980,7 +1096,10 @@ async function finalizeWorkflow( failureReasons: string[], canResolveStale: boolean, gate: ReviewFeedbackGate, - triggerErrors: string[] + triggerErrors: string[], + matchedTriggers: ResolvedTrigger[], + resolvedTriggers: ResolvedTrigger[], + inputs: ActionInputs ): Promise { await dismissPreviousReviewIfResolved( octokit, @@ -996,14 +1115,15 @@ async function finalizeWorkflow( setWorkflowOutputs(outputs); // Write structured findings to file for external export (GCS, S3, etc.) - try { - const findingsPath = writeFindingsOutput(reports, context, findingObservations, { + writeFindingsOutputs( + () => writeFindingsOutput(reports, context, findingObservations, { triggerResults: toReplayTriggerResults(results), - }); - logAction(`Findings written to ${findingsPath}`); - } catch (error) { - warnAction(`Failed to write findings output: ${error}`); - } + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }), + () => writeSchemaV2Outputs(inputs, context, resolvedTriggers, matchedTriggers, results, findingObservations, warnAction), + warnAction, + (path) => logAction(`Findings written to ${path}`) + ); // Update core check with overall summary if (coreCheckId && context.pullRequest) { @@ -1187,9 +1307,13 @@ async function runOrFailCore( } } -function resolveFindingsFilePath(inputPath: string | undefined, repoPath: string): string { +function resolveFindingsFilePath( + inputPath: string | undefined, + repoPath: string, + missingMessage = 'findings-file is required when mode is report' +): string { if (!inputPath) { - setFailed('findings-file is required when mode is report'); + setFailed(missingMessage); } return isAbsolute(inputPath) ? inputPath : join(repoPath, inputPath); } @@ -1280,6 +1404,23 @@ function toReplayTriggerResults(results: TriggerResult[]): ReplayTriggerResult[] })); } +/** The trigger-derived fields common to every report-mode result, before the found/error/report outcome is layered on. */ +function buildReplayBaseResult(trigger: ResolvedTrigger, inputs: ActionInputs) { + return { + triggerId: trigger.id, + triggerName: trigger.name, + skillName: trigger.skill, + skillExecutionId: trigger.skillExecutionId, + failOn: trigger.failOn ?? inputs.failOn, + reportOn: trigger.reportOn ?? inputs.reportOn, + minConfidence: trigger.minConfidence ?? 'medium', + reportOnSuccess: trigger.reportOnSuccess, + requestChanges: trigger.requestChanges ?? inputs.requestChanges, + failCheck: trigger.failCheck ?? inputs.failCheck, + maxFindings: trigger.maxFindings ?? inputs.maxFindings, + }; +} + /** * Rebuild report-mode trigger results by joining artifact rows to the current * configured trigger name and skill identity. @@ -1343,24 +1484,7 @@ function buildReportModeResults( } const results = matchedTriggers.map((trigger) => { - const failOn = trigger.failOn ?? inputs.failOn; - const reportOn = trigger.reportOn ?? inputs.reportOn; - const minConfidence = trigger.minConfidence ?? 'medium'; - const requestChanges = trigger.requestChanges ?? inputs.requestChanges; - const failCheck = trigger.failCheck ?? inputs.failCheck; - const maxFindings = trigger.maxFindings ?? inputs.maxFindings; - const baseResult = { - triggerId: trigger.id, - triggerName: trigger.name, - skillName: trigger.skill, - failOn, - reportOn, - minConfidence, - reportOnSuccess: trigger.reportOnSuccess, - requestChanges, - failCheck, - maxFindings, - }; + const baseResult = buildReplayBaseResult(trigger, inputs); const outputResult = outputResults.get(triggerReplayKey(trigger))?.shift() ?? outputResults.get(resultKey(trigger.name, trigger.skill))?.shift(); @@ -1447,17 +1571,17 @@ async function createCompletedSkillChecksForReport( minConfidence: result.minConfidence, failCheck: result.failCheck, }); - updatedResults.push(withRenderedReviewResult({ ...result, checkRunUrl: check.url })); + updatedResults.push(withRenderedReviewResult({ ...result, checkRunUrl: check.url, checkRunId: check.checkRunId })); continue; } - await createFailedSkillCheck( + const failedCheck = await createFailedSkillCheck( octokit, result.skillName, result.error ?? new Error('Trigger did not produce a report'), options ); - updatedResults.push(result); + updatedResults.push({ ...result, checkRunUrl: failedCheck.url, checkRunId: failedCheck.checkRunId }); } return updatedResults; @@ -1572,7 +1696,14 @@ async function finalizeReportWorkflow( canResolveStale: boolean, gate: ReviewFeedbackGate, triggerErrors: string[], - options: { failOnWriteError?: boolean } = {} + options: { + failOnWriteError?: boolean; + matchedTriggers: ResolvedTrigger[]; + resolvedTriggers: ResolvedTrigger[]; + inputs: ActionInputs; + metadataOutputV2?: WardenMetadata; + findingsOutputV2?: WardenFindingsV2; + } ): Promise { await dismissPreviousReviewIfResolved( octokit, @@ -1587,14 +1718,21 @@ async function finalizeReportWorkflow( const outputs = computeWorkflowOutputs(reports); setWorkflowOutputs(outputs); - try { - const findingsPath = writeFindingsOutput(reports, context, findingObservations, { + writeFindingsOutputs( + () => writeFindingsOutput(reports, context, findingObservations, { triggerResults: toReplayTriggerResults(results), - }); - logAction(`Findings written to ${findingsPath}`); - } catch (error) { - warnAction(`Failed to write findings output: ${error}`); - } + configuredSkills: buildConfiguredSkillsList({ + allTriggers: options.resolvedTriggers, + matchedTriggers: options.matchedTriggers, + }), + }), + () => writeSchemaV2ReportOutputs( + options.metadataOutputV2, options.findingsOutputV2, context, results, + options.matchedTriggers, findingObservations, warnAction + ), + warnAction, + (path) => logAction(`Findings written to ${path}`) + ); await createCompletedCoreCheckForReport( octokit, @@ -1663,7 +1801,7 @@ async function cleanupOrphanedComments( const { allResolved, autoResolvedByFixEvaluation, autoResolvedByStaleCheck, findingObservations } = await evaluateFixesAndResolveStale( - octokit, context, existingComments, [], new Set(), true, inputs.anthropicApiKey, auxiliaryOptions, gate, { + octokit, context, existingComments, [], new Set(), true, inputs.anthropicApiKey, auxiliaryOptions, gate, [], { failOnWriteError: options.failOnWriteError, } ); @@ -1712,6 +1850,7 @@ async function runAnalyzeMode( const { context, runnerConcurrency, + resolvedTriggers, matchedTriggers, skipCoreCheck, } = initResult; @@ -1720,23 +1859,32 @@ async function runAnalyzeMode( setOutput('findings-count', 0); setOutput('high-count', 0); setOutput('summary', skipCoreCheck?.title ?? 'No triggers matched'); - try { - const findingsPath = writeFindingsOutput([], context, [], { triggerResults: [] }); - logAction(`Findings written to ${findingsPath}`); - } catch (error) { - setFailed(`Failed to write findings output: ${error}`); - } + writeFindingsOutputs( + () => writeFindingsOutput([], context, [], { + triggerResults: [], + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }), + () => writeSchemaV2Outputs(inputs, context, resolvedTriggers, matchedTriggers, [], [], setFailed), + setFailed, + (path) => logAction(`Findings written to ${path}`) + ); logAction('Analysis complete: 0 total findings'); return; } + const liveResults: TriggerResult[] = []; const results = await Sentry.startSpan( { op: 'workflow.execute', name: 'execute triggers', attributes: { 'warden.trigger.count': matchedTriggers.length }, }, - () => executeAllTriggers(matchedTriggers, context, runnerConcurrency, inputs), + () => executeAllTriggers(matchedTriggers, context, runnerConcurrency, inputs, { + onTriggerComplete: (result) => { + liveResults.push(result); + writeSchemaV2OutputsLive(inputs, context, resolvedTriggers, matchedTriggers, liveResults); + }, + }), ); const reports = results.flatMap((result) => (result.report ? [result.report] : [])); @@ -1744,19 +1892,181 @@ async function runAnalyzeMode( setWorkflowOutputs(outputs); span.setAttribute('warden.finding.count', reports.flatMap((r) => r.findings).length); - try { - const findingsPath = writeFindingsOutput(reports, context, [], { + writeFindingsOutputs( + () => writeFindingsOutput(reports, context, [], { triggerResults: toReplayTriggerResults(results), - }); - logAction(`Findings written to ${findingsPath}`); - } catch (error) { - setFailed(`Failed to write findings output: ${error}`); - } + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }), + () => writeSchemaV2Outputs(inputs, context, resolvedTriggers, matchedTriggers, results, [], setFailed), + setFailed, + (path) => logAction(`Findings written to ${path}`) + ); handleTriggerErrors(collectTriggerErrors(results), matchedTriggers.length, { failAll: false }); logAction(`Analysis complete: ${outputs.findingsCount} total findings`); } +function readMetadataFileV2(inputPath: string | undefined, repoPath: string): WardenMetadata { + const filePath = resolveFindingsFilePath( + inputPath, repoPath, + 'metadata-file is required when mode is report and output-schema-version is \'2\'' + ); + + try { + return WardenMetadataSchema.parse(JSON.parse(readFileSync(filePath, 'utf-8'))); + } catch (error) { + setFailed(`Failed to read metadata file ${filePath}: ${error}`); + } +} + +function readFindingsFileV2(inputPath: string | undefined, repoPath: string): WardenFindingsV2 { + const filePath = resolveFindingsFilePath(inputPath, repoPath); + + try { + return WardenFindingsSchemaV2.parse(JSON.parse(readFileSync(filePath, 'utf-8'))); + } catch (error) { + setFailed(`Failed to read findings file ${filePath}: ${error}`); + } +} + +function validateV2OutputsMatchContext( + metadata: WardenMetadata, + findings: WardenFindingsV2, + context: EventContext +): void { + if (metadata.runId !== findings.runId || metadata.schemaVersion !== findings.schemaVersion) { + setFailed('Metadata file and findings file do not share the same runId/schemaVersion'); + } + + if (metadata.repository.fullName !== context.repository.fullName) { + setFailed( + `Metadata file is for ${metadata.repository.fullName}, but this workflow is for ${context.repository.fullName}` + ); + } + + if (metadata.event !== context.eventType) { + setFailed(`Metadata file event ${metadata.event} does not match ${context.eventType}`); + } + + if (!context.pullRequest) { + return; + } + + if (!metadata.pullRequest) { + setFailed('Metadata file is missing pull request metadata'); + } + + if (metadata.pullRequest.number !== context.pullRequest.number) { + setFailed( + `Metadata file is for PR #${metadata.pullRequest.number}, but this workflow is for PR #${context.pullRequest.number}` + ); + } + + if (metadata.pullRequest.headSha !== context.pullRequest.headSha) { + setFailed( + `Metadata file head SHA ${metadata.pullRequest.headSha} does not match current head SHA ${context.pullRequest.headSha}` + ); + } +} + +/** + * Rebuild report-mode trigger results from schema-v2 metadata/findings artifacts, + * mirroring buildReportModeResults's v1 join-by-trigger-identity behavior. + */ +function buildReportModeResultsV2( + metadata: WardenMetadata, + findingsOutput: WardenFindingsV2, + matchedTriggers: ResolvedTrigger[], + inputs: ActionInputs +): TriggerResult[] { + const executionsWithTriggerId = findingsOutput.skillExecutions.filter((execution) => execution.triggerId); + const duplicateExecutionTriggerIds = new Set(); + const seenExecutionTriggerIds = new Set(); + for (const execution of executionsWithTriggerId) { + const triggerId = execution.triggerId as string; + if (seenExecutionTriggerIds.has(triggerId)) { + duplicateExecutionTriggerIds.add(triggerId); + } + seenExecutionTriggerIds.add(triggerId); + } + if (duplicateExecutionTriggerIds.size > 0) { + const skillNames = executionsWithTriggerId + .filter((execution) => duplicateExecutionTriggerIds.has(execution.triggerId as string)) + .map((execution) => execution.skillName) + .join(', '); + throw new Error(`Findings file contains ambiguous duplicate trigger result(s): ${skillNames}`); + } + + const executionsByTriggerId = new Map( + executionsWithTriggerId.map((execution) => [execution.triggerId as string, execution]) + ); + const findingsByExecutionScopedId = new Map( + findingsOutput.findings.map((finding) => [`${finding.provenance.originSkillExecutionId}:${finding.id}`, finding]) + ); + const errorByTriggerId = new Map( + (metadata.triggerResults ?? []) + .filter((result) => result.status === 'error' && result.triggerId) + .map((result) => [result.triggerId as string, (result as { error: { name?: string; message: string } }).error]) + ); + + const consumedTriggerIds = new Set(); + + const results = matchedTriggers.map((trigger) => { + const baseResult = buildReplayBaseResult(trigger, inputs); + + const execution = executionsByTriggerId.get(trigger.id); + if (!execution) { + const error = errorByTriggerId.get(trigger.id); + return { + ...baseResult, + error: error + ? deserializeTriggerError(error, `Trigger ${trigger.name} (${trigger.skill}) failed during analysis`) + : new Error(`Findings file has no result for trigger ${trigger.name} (${trigger.skill})`), + }; + } + consumedTriggerIds.add(trigger.id); + + const findings = execution.findingIds.flatMap((id) => { + const finding = findingsByExecutionScopedId.get(`${execution.skillExecutionId}:${id}`); + return finding ? [toFindingFromV2(finding)] : []; + }); + + const { usage: auxiliaryUsage, attribution: auxiliaryUsageAttribution } = fromAuxiliaryUsageEntries( + execution.auxiliaryUsage + ); + + const report: SkillReport = { + skill: execution.skillName, + summary: execution.summary, + findings, + durationMs: execution.durationMs, + usage: execution.usage, + auxiliaryUsage, + auxiliaryUsageAttribution, + failedHunks: execution.failedHunks, + failedExtractions: execution.failedExtractions, + error: execution.error, + verifierRejections: execution.verifierRejections, + model: execution.model, + runtime: execution.runtime, + }; + + return { ...baseResult, report }; + }); + + const unreportedExecutions = executionsWithTriggerId.filter( + (execution) => !consumedTriggerIds.has(execution.triggerId as string) + ); + if (unreportedExecutions.length > 0) { + const skillNames = unreportedExecutions.map((execution) => execution.skillName).join(', '); + throw new Error( + `Findings file contains ${unreportedExecutions.length} result(s) that do not match current config: ${skillNames}` + ); + } + + return results; +} + /** * Run the reporting phase without rerunning skills. * It replays analyze output against the current PR config and owns GitHub writes. @@ -1771,12 +2081,22 @@ async function runReportMode( const { context, auxiliaryOptions, + resolvedTriggers, matchedTriggers, skippedTriggers, skipCoreCheck, } = initResult; - const findingsOutput = readFindingsFile(inputs.findingsFile, repoPath); - validateFindingsMatchContext(findingsOutput, context); + let metadataOutputV2: WardenMetadata | undefined; + let findingsOutputV2: WardenFindingsV2 | undefined; + let findingsOutputV1: FindingsOutput | undefined; + if (inputs.outputSchemaVersion === '2') { + metadataOutputV2 = readMetadataFileV2(inputs.metadataFile, repoPath); + findingsOutputV2 = readFindingsFileV2(inputs.findingsFile, repoPath); + validateV2OutputsMatchContext(metadataOutputV2, findingsOutputV2, context); + } else { + findingsOutputV1 = readFindingsFile(inputs.findingsFile, repoPath); + validateFindingsMatchContext(findingsOutputV1, context); + } let results: TriggerResult[] = []; let previousReviewInfo: BotReviewInfo | null = null; @@ -1785,18 +2105,23 @@ async function runReportMode( let canResolveStale!: boolean; try { - results = buildReportModeResults(findingsOutput, matchedTriggers, inputs); + results = metadataOutputV2 && findingsOutputV2 + ? buildReportModeResultsV2(metadataOutputV2, findingsOutputV2, matchedTriggers, inputs) + : buildReportModeResults(findingsOutputV1 as FindingsOutput, matchedTriggers, inputs); await createCompletedSkippedSkillChecks(octokit, context, skippedTriggers); if (skipCoreCheck) { const outputs = { findingsCount: 0, highCount: 0, summary: skipCoreCheck.title }; setWorkflowOutputs(outputs); - try { - const findingsPath = writeFindingsOutput([], context, [], { triggerResults: [] }); - logAction(`Findings written to ${findingsPath}`); - } catch (error) { - warnAction(`Failed to write findings output: ${error}`); - } + writeFindingsOutputs( + () => writeFindingsOutput([], context, [], { + triggerResults: [], + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }), + () => writeSchemaV2ReportOutputs(metadataOutputV2, findingsOutputV2, context, [], matchedTriggers, [], warnAction), + warnAction, + (path) => logAction(`Findings written to ${path}`) + ); await createCompletedCoreCheckForReport( octokit, context, @@ -1824,14 +2149,17 @@ async function runReportMode( ); const outputs = { findingsCount: 0, highCount: 0, summary: 'No triggers matched' }; setWorkflowOutputs(outputs); - try { - const findingsPath = writeFindingsOutput([], context, cleanupFindingObservations, { + writeFindingsOutputs( + () => writeFindingsOutput([], context, cleanupFindingObservations, { triggerResults: [], - }); - logAction(`Findings written to ${findingsPath}`); - } catch (error) { - warnAction(`Failed to write findings output: ${error}`); - } + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }), + () => writeSchemaV2ReportOutputs( + metadataOutputV2, findingsOutputV2, context, [], matchedTriggers, cleanupFindingObservations, warnAction + ), + warnAction, + (path) => logAction(`Findings written to ${path}`) + ); await createCompletedCoreCheckForReport( octokit, context, @@ -1876,7 +2204,7 @@ async function runReportMode( octokit, context, reviewPhase.fetchedComments, allFindings, reviewPhase.activeWardenCommentIds, canResolveStale, inputs.anthropicApiKey, - auxiliaryOptions, gate, + auxiliaryOptions, gate, matchedTriggers, { failOnWriteError: true }, ); resolveSpan.setAttribute( @@ -1899,7 +2227,7 @@ async function runReportMode( canResolveStale, gate, triggerErrors, - { failOnWriteError: true }, + { failOnWriteError: true, matchedTriggers, resolvedTriggers, inputs, metadataOutputV2, findingsOutputV2 }, ); } catch (error) { if (error instanceof ActionFailedError) { @@ -1938,6 +2266,7 @@ export async function runPRWorkflow( context, runnerConcurrency, auxiliaryOptions, + resolvedTriggers, matchedTriggers, skippedTriggers, skipCoreCheck, @@ -1987,11 +2316,13 @@ export async function runPRWorkflow( setOutput('findings-count', 0); setOutput('high-count', 0); setOutput('summary', skipCoreCheck.title); - try { - writeFindingsOutput([], context); - } catch (error) { - warnAction(`Failed to write findings output: ${error}`); - } + writeFindingsOutputs( + () => writeFindingsOutput([], context, [], { + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }), + () => writeSchemaV2Outputs(inputs, context, resolvedTriggers, matchedTriggers, [], [], warnAction), + warnAction + ); await completeSkippedCoreCheck(octokit, context, coreCheckId, skipCoreCheck); return; } @@ -2007,11 +2338,15 @@ export async function runPRWorkflow( setOutput('findings-count', 0); setOutput('high-count', 0); setOutput('summary', 'No triggers matched'); - try { - writeFindingsOutput([], context, cleanupFindingObservations); - } catch (error) { - warnAction(`Failed to write findings output: ${error}`); - } + writeFindingsOutputs( + () => writeFindingsOutput([], context, cleanupFindingObservations, { + configuredSkills: buildConfiguredSkillsList({ allTriggers: resolvedTriggers, matchedTriggers }), + }), + () => writeSchemaV2Outputs( + inputs, context, resolvedTriggers, matchedTriggers, [], cleanupFindingObservations, warnAction + ), + warnAction + ); await completeSkippedCoreCheck(octokit, context, coreCheckId, { title: 'No triggers matched', message: 'No triggers matched for this event.', @@ -2021,6 +2356,7 @@ export async function runPRWorkflow( } let results: TriggerResult[]; + const liveResults: TriggerResult[] = []; try { results = await Sentry.startSpan( { @@ -2030,6 +2366,10 @@ export async function runPRWorkflow( }, () => executeAllTriggers(matchedTriggers, context, runnerConcurrency, inputs, { checks: createTriggerCheckReporter(octokit, context), + onTriggerComplete: (result) => { + liveResults.push(result); + writeSchemaV2OutputsLive(inputs, context, resolvedTriggers, matchedTriggers, liveResults); + }, }), ); } catch (error) { @@ -2065,7 +2405,7 @@ export async function runPRWorkflow( octokit, context, reviewPhase.fetchedComments, allFindings, reviewPhase.activeWardenCommentIds, canResolveStale, inputs.anthropicApiKey, - auxiliaryOptions, gate, + auxiliaryOptions, gate, matchedTriggers, ); resolveSpan.setAttribute( 'warden.feedback.auto_resolve.fix_eval_count', @@ -2088,6 +2428,9 @@ export async function runPRWorkflow( canResolveStale, gate, triggerErrors, + matchedTriggers, + resolvedTriggers, + inputs, ); handleTriggerErrors(triggerErrors, matchedTriggers.length); diff --git a/packages/warden/src/action/workflow/schedule.test.ts b/packages/warden/src/action/workflow/schedule.test.ts index f8943cbd1..6304f156c 100644 --- a/packages/warden/src/action/workflow/schedule.test.ts +++ b/packages/warden/src/action/workflow/schedule.test.ts @@ -1,9 +1,11 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { join } from 'node:path'; +import { readFileSync, rmSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import type { Octokit } from '@octokit/rest'; import type { ActionInputs } from '../inputs.js'; import type { SkillReport, Finding, EventContext } from '../../types/index.js'; +import { getMetadataOutputPath, getFindingsOutputPathV2, getFindingsOutputPath } from './base.js'; // ----------------------------------------------------------------------------- // Fixtures Directory @@ -13,6 +15,7 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url)); const SCHEDULE_FIXTURES = join(__dirname, '__fixtures__/schedule'); const SCHEDULE_BASE_ONLY_FIXTURES = join(__dirname, '__fixtures__/schedule-base-only'); const SCHEDULE_MULTI_FIXTURES = join(__dirname, '__fixtures__/schedule-multi'); +const SCHEDULE_WITH_PR_SKILL_FIXTURES = join(__dirname, '__fixtures__/schedule-with-pr-skill'); const SCHEDULE_TITLE_FIXTURES = join(__dirname, '__fixtures__/schedule-title'); const NO_CONFIG_FIXTURES = join(__dirname, '__fixtures__/no-config'); const RUNTIME_CLAUDE_FIXTURES = join(__dirname, '__fixtures__/runtime-claude'); @@ -32,6 +35,8 @@ vi.mock('./base.js', async () => { return { ...actual, setFailed: mockedSetFailed, + writeFindingsOutput: vi.fn(actual['writeFindingsOutput'] as (...args: unknown[]) => unknown), + writeSchemaV2OutputLive: vi.fn(actual['writeSchemaV2OutputLive'] as (...args: unknown[]) => unknown), ensureClaudeAuth: vi.fn((inputs: ActionInputs): void => { if (inputs.anthropicApiKey || inputs.oauthToken) { return; @@ -101,7 +106,8 @@ import { runSkill } from '../../sdk/runner.js'; import { buildScheduleEventContext } from '../../event/schedule-context.js'; import { createOrUpdateIssue } from '../../output/github-issues.js'; import { resolveSkillAsync } from '../../skills/loader.js'; -import { setFailed } from './base.js'; +import { setFailed, writeFindingsOutput, writeSchemaV2OutputLive } from './base.js'; +import { buildMetadataOutputV2 } from '../reporting/output-v2.js'; import { runScheduleWorkflow } from './schedule.js'; import { clearSkillsCache } from '../../skills/loader.js'; @@ -137,6 +143,7 @@ function createDefaultInputs(overrides: Partial = {}): ActionInput configPath: 'warden.toml', maxFindings: 50, parallel: 2, + outputSchemaVersion: '1', ...overrides, }; } @@ -444,6 +451,41 @@ describe('runScheduleWorkflow', () => { expect(mockCreateOrUpdateIssue).toHaveBeenCalledTimes(1); }); + it('includes configuredSkills in the v1 findings output, matching the PR workflow', async () => { + mockRunSkill.mockResolvedValue(createSkillReport({ findings: [] })); + + const findingsFile = getFindingsOutputPath(SCHEDULE_FIXTURES); + + try { + await runScheduleWorkflow(mockOctokit, createDefaultInputs(), SCHEDULE_FIXTURES); + + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + expect(findings.configuredSkills).toEqual([{ name: 'test-skill', triggered: true }]); + } finally { + rmSync(findingsFile, { force: true }); + } + }); + + it('lists PR-only skills as untriggered in configuredSkills instead of omitting them', async () => { + mockRunSkill.mockResolvedValue(createSkillReport({ findings: [] })); + + const findingsFile = getFindingsOutputPath(SCHEDULE_WITH_PR_SKILL_FIXTURES); + + try { + await runScheduleWorkflow(mockOctokit, createDefaultInputs(), SCHEDULE_WITH_PR_SKILL_FIXTURES); + + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + expect(findings.configuredSkills).toEqual( + expect.arrayContaining([ + { name: 'test-skill', triggered: true }, + { name: 'pr-only-skill', triggered: false }, + ]) + ); + } finally { + rmSync(findingsFile, { force: true }); + } + }); + it('skips skill run when no files match trigger', async () => { mockBuildContext.mockResolvedValue( createScheduleContext({ @@ -468,6 +510,318 @@ describe('runScheduleWorkflow', () => { }); }); + // --------------------------------------------------------------------------- + // Schema v2 output + // --------------------------------------------------------------------------- + + describe('schema v2 output', () => { + it('writes schema-v2 metadata and findings for a matched schedule trigger', async () => { + const finding = createFinding({ severity: 'high' }); + mockRunSkill.mockResolvedValue(createSkillReport({ findings: [finding] })); + + const metadataFile = getMetadataOutputPath(SCHEDULE_FIXTURES); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_FIXTURES); + + try { + await runScheduleWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + SCHEDULE_FIXTURES + ); + + const metadata = JSON.parse(readFileSync(metadataFile, 'utf-8')); + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + + expect(metadata.schemaVersion).toBe('2'); + expect(findings.schemaVersion).toBe('2'); + expect(findings.skillExecutions).toHaveLength(1); + expect(findings.findings).toHaveLength(1); + expect(findings.findings[0]?.id).toBe(finding.id); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('links the schedule-created GitHub issue onto the skill execution in v2 output', async () => { + mockRunSkill.mockResolvedValue(createSkillReport({ findings: [createFinding({ severity: 'high' })] })); + mockCreateOrUpdateIssue.mockResolvedValue({ + issueNumber: 42, + issueUrl: 'https://github.com/test-owner/test-repo/issues/42', + created: true, + }); + + const metadataFile = getMetadataOutputPath(SCHEDULE_FIXTURES); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_FIXTURES); + + try { + await runScheduleWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + SCHEDULE_FIXTURES + ); + + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + + expect(findings.skillExecutions[0]?.issueNumber).toBe(42); + expect(findings.skillExecutions[0]?.issueUrl).toBe('https://github.com/test-owner/test-repo/issues/42'); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('keeps the real report in v2 output when creating the tracking issue fails', async () => { + const finding = createFinding({ severity: 'high' }); + mockRunSkill.mockResolvedValue(createSkillReport({ findings: [finding] })); + mockCreateOrUpdateIssue.mockRejectedValue(new Error('rate limited')); + + const metadataFile = getMetadataOutputPath(SCHEDULE_FIXTURES); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_FIXTURES); + + try { + await runScheduleWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + SCHEDULE_FIXTURES + ); + + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + + expect(findings.skillExecutions).toHaveLength(1); + expect(findings.skillExecutions[0]?.error).toBeUndefined(); + expect(findings.skillExecutions[0]?.issueNumber).toBeUndefined(); + expect(findings.findings).toHaveLength(1); + expect(findings.findings[0]?.id).toBe(finding.id); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('Failed to create/update issue')); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('records verification provenance from onFindingProcessing in schedule v2 output', async () => { + const finding = createFinding({ severity: 'high' }); + mockRunSkill.mockImplementation(async (_skill, _context, options) => { + options?.callbacks?.onFindingProcessing?.({ + stage: 'verification', + action: 'kept', + finding, + reason: 'still real after tracing', + model: 'claude-haiku-4-5', + }); + return createSkillReport({ findings: [finding] }); + }); + + const metadataFile = getMetadataOutputPath(SCHEDULE_FIXTURES); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_FIXTURES); + + try { + await runScheduleWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + SCHEDULE_FIXTURES + ); + + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + expect(findings.findings[0]?.provenance.verification).toEqual({ + outcome: 'kept', + model: 'claude-haiku-4-5', + runtime: undefined, + }); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('records a schedule trigger with no matching files as skipped with reason no_changes', async () => { + mockBuildContext.mockResolvedValue( + createScheduleContext({ + pullRequest: { + number: 0, + title: 'Scheduled Analysis', + body: null, + author: 'warden', + baseBranch: 'main', + headBranch: 'main', + headSha: 'abc123', + baseSha: 'abc123', + files: [], + }, + }) + ); + + const metadataFile = getMetadataOutputPath(SCHEDULE_FIXTURES); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_FIXTURES); + + try { + await runScheduleWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + SCHEDULE_FIXTURES + ); + + const metadata = JSON.parse(readFileSync(metadataFile, 'utf-8')); + + expect(metadata.skippedTriggers).toEqual( + expect.arrayContaining([expect.objectContaining({ reason: 'no_changes' })]) + ); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('does not write schema-v2 files when output-schema-version is 1', async () => { + mockRunSkill.mockResolvedValue(createSkillReport({ findings: [] })); + + const metadataFile = getMetadataOutputPath(SCHEDULE_FIXTURES); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_FIXTURES); + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + + await runScheduleWorkflow(mockOctokit, createDefaultInputs(), SCHEDULE_FIXTURES); + + expect(() => readFileSync(metadataFile, 'utf-8')).toThrow(); + expect(() => readFileSync(findingsFile, 'utf-8')).toThrow(); + }); + + it('still writes schema-v2 files on the no-config early return when the v1 write throws', async () => { + vi.mocked(writeFindingsOutput).mockImplementationOnce(() => { + throw new Error('disk full'); + }); + + const metadataFile = getMetadataOutputPath(NO_CONFIG_FIXTURES); + const findingsFile = getFindingsOutputPathV2(NO_CONFIG_FIXTURES); + + try { + await runScheduleWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + NO_CONFIG_FIXTURES + ); + + const metadata = JSON.parse(readFileSync(metadataFile, 'utf-8')); + expect(metadata.schemaVersion).toBe('2'); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('still writes schema-v2 files when every schedule trigger fails', async () => { + mockRunSkill.mockRejectedValue(new Error('Skill failed')); + + const metadataFile = getMetadataOutputPath(SCHEDULE_MULTI_FIXTURES); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_MULTI_FIXTURES); + + try { + await expect( + runScheduleWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + SCHEDULE_MULTI_FIXTURES + ) + ).rejects.toThrow('setFailed'); + + const metadata = JSON.parse(readFileSync(metadataFile, 'utf-8')); + const findings = JSON.parse(readFileSync(findingsFile, 'utf-8')); + expect(metadata.schemaVersion).toBe('2'); + expect(findings.schemaVersion).toBe('2'); + expect(metadata.triggerResults).toHaveLength(2); + expect(metadata.triggerResults.every((result: { status: string }) => result.status === 'error')).toBe(true); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('reports a not-yet-run trigger as pending during a live write, then replaces it with a real result', async () => { + let invocation = 0; + let releaseSecond!: () => void; + const secondGate = new Promise((resolve) => { + releaseSecond = resolve; + }); + + mockRunSkill.mockImplementation(async () => { + invocation++; + if (invocation === 2) await secondGate; + return createSkillReport({ findings: [] }); + }); + + const metadataFile = getMetadataOutputPath(SCHEDULE_MULTI_FIXTURES); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_MULTI_FIXTURES); + + const workflow = runScheduleWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + SCHEDULE_MULTI_FIXTURES + ); + + try { + await vi.waitFor(() => { + const metadata = JSON.parse(readFileSync(metadataFile, 'utf-8')); + expect(metadata.triggerResults).toHaveLength(1); + }); + + const midMetadata = JSON.parse(readFileSync(metadataFile, 'utf-8')); + expect(midMetadata.skippedTriggers).toHaveLength(1); + expect(midMetadata.skippedTriggers[0].reason).toBe('pending'); + + releaseSecond(); + await workflow; + + const finalMetadata = JSON.parse(readFileSync(metadataFile, 'utf-8')); + expect(finalMetadata.triggerResults).toHaveLength(2); + expect(finalMetadata.skippedTriggers ?? []).toHaveLength(0); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + + it('reports a non-schedule trigger as no_event_match during a live write, never as pending', async () => { + mockRunSkill.mockResolvedValue(createSkillReport({ findings: [] })); + + const metadataFile = getMetadataOutputPath(SCHEDULE_WITH_PR_SKILL_FIXTURES); + const findingsFile = getFindingsOutputPathV2(SCHEDULE_WITH_PR_SKILL_FIXTURES); + + try { + await runScheduleWorkflow( + mockOctokit, + createDefaultInputs({ outputSchemaVersion: '2' }), + SCHEDULE_WITH_PR_SKILL_FIXTURES + ); + + const liveCalls = vi.mocked(writeSchemaV2OutputLive).mock.calls; + expect(liveCalls.length).toBeGreaterThan(0); + + // The first live write fires right after the only schedule trigger + // completes, while the pull_request-type trigger is still unprocessed + // by this loop — exactly the state that used to mislabel it 'pending'. + const [context, resolvedTriggers, matchedTriggers, results, , options] = liveCalls[0]!; + const liveMetadata = buildMetadataOutputV2(context, resolvedTriggers, matchedTriggers, results, options); + + const prOnlySkipped = liveMetadata.skippedTriggers?.find((t) => t.skillName === 'pr-only-skill'); + expect(prOnlySkipped?.reason).toBe('no_event_match'); + } finally { + rmSync(metadataFile, { force: true }); + rmSync(findingsFile, { force: true }); + rmSync(`${findingsFile}.done`, { force: true }); + } + }); + }); + // --------------------------------------------------------------------------- // Issue & PR Creation // --------------------------------------------------------------------------- diff --git a/packages/warden/src/action/workflow/schedule.ts b/packages/warden/src/action/workflow/schedule.ts index 1d214fbe0..6fb1b474b 100644 --- a/packages/warden/src/action/workflow/schedule.ts +++ b/packages/warden/src/action/workflow/schedule.ts @@ -16,11 +16,11 @@ import type { ScheduleConfig } from '../../config/schema.js'; import { buildScheduleEventContext } from '../../event/schedule-context.js'; import { runSkill } from '../../sdk/runner.js'; import { assertValidPiModelSelectors } from '../../sdk/runtimes/model-selectors.js'; -import { createOrUpdateIssue } from '../../output/github-issues.js'; +import { createOrUpdateIssue, type IssueResult } from '../../output/github-issues.js'; import { shouldFail, countFindingsAtOrAbove, countSeverity } from '../../triggers/matcher.js'; import { resolveSkillAsync } from '../../skills/loader.js'; import { filterFindings } from '../../types/index.js'; -import type { SkillReport } from '../../types/index.js'; +import type { EventContext, SkillReport } from '../../types/index.js'; import { Sentry, logger, setRepositoryScope, emitRunMetric } from '../../sentry.js'; import type { ActionInputs } from '../inputs.js'; import { @@ -33,7 +33,14 @@ import { handleTriggerErrors, getDefaultBranchFromAPI, writeFindingsOutput, + writeFindingsOutputs, + writeSchemaV2Output, + writeSchemaV2OutputLive, + buildV2WriteOptions, } from './base.js'; +import type { TriggerResult } from '../triggers/executor.js'; +import type { FindingProcessingEvent } from '../../sdk/types.js'; +import { buildConfiguredSkillsList } from '../reporting/output.js'; import { captureActionTriggerError } from '../error-reporting.js'; // ----------------------------------------------------------------------------- @@ -45,6 +52,54 @@ interface WorkflowSpan { spanContext?: () => { traceId: string }; } +function writeSchemaV2ScheduleOutputs( + inputs: ActionInputs, + context: EventContext, + resolvedTriggers: ResolvedTrigger[], + matchedTriggers: ResolvedTrigger[], + results: TriggerResult[] +): void { + if (inputs.outputSchemaVersion !== '2') return; + + try { + const { metadataPath, findingsPath } = writeSchemaV2Output( + context, resolvedTriggers, matchedTriggers, results, [], + buildV2WriteOptions(inputs) + ); + console.log(`Metadata written to ${metadataPath}`); + console.log(`Findings (v2) written to ${findingsPath}`); + } catch (error) { + console.error(`::warning::Failed to write schema-v2 output: ${error}`); + } +} + +/** + * Live counterpart of {@link writeSchemaV2ScheduleOutputs}, fired after each + * trigger completes. Unlike `pr-workflow.ts`, `matchedTriggers` here grows + * incrementally inside the same sequential loop this is called from, so a + * trigger this loop simply hasn't reached yet must be reported as `'pending'` + * rather than a guessed skip reason. + */ +function writeSchemaV2ScheduleOutputsLive( + inputs: ActionInputs, + context: EventContext, + resolvedTriggers: ResolvedTrigger[], + matchedTriggers: ResolvedTrigger[], + results: TriggerResult[], + processedTriggerIds: Set +): void { + if (inputs.outputSchemaVersion !== '2') return; + const pendingTriggerIds = new Set( + resolvedTriggers + .filter((t) => t.type === 'schedule' && !processedTriggerIds.has(t.id)) + .map((t) => t.id) + ); + writeSchemaV2OutputLive(context, resolvedTriggers, matchedTriggers, results, [], { + ...buildV2WriteOptions(inputs), + pendingTriggerIds, + }); +} + export async function runScheduleWorkflow( octokit: Octokit, inputs: ActionInputs, @@ -76,6 +131,7 @@ async function runScheduleWorkflowInner( logGroupEnd(); let scheduleTriggers: ResolvedTrigger[]; + let allResolvedTriggers: ResolvedTrigger[]; let skillRootsByName: LayeredSkillRootsByName | undefined; try { const layered = loadLayeredWardenConfig(repoPath, { @@ -84,8 +140,8 @@ async function runScheduleWorkflowInner( onWarning: (message) => console.log(`::warning::${message}`), }); skillRootsByName = buildSkillRootsByName(repoPath, layered, inputs.baseSkillRoot); - scheduleTriggers = resolveLayeredSkillConfigs(layered, undefined, skillRootsByName) - .filter((t) => t.type === 'schedule'); + allResolvedTriggers = resolveLayeredSkillConfigs(layered, undefined, skillRootsByName); + scheduleTriggers = allResolvedTriggers.filter((t) => t.type === 'schedule'); } catch (error) { if ( error instanceof ConfigLoadError && @@ -96,20 +152,21 @@ async function runScheduleWorkflowInner( setOutput('findings-count', 0); setOutput('high-count', 0); setOutput('summary', 'No warden.toml found'); - try { - const fullName = process.env['GITHUB_REPOSITORY'] ?? ''; - const [o = '', n = ''] = fullName.split('/'); - workflowSpan.setAttribute('warden.trigger.count', 0); - workflowSpan.setAttribute('warden.finding.count', 0); - writeFindingsOutput([], { - eventType: 'schedule', - action: 'scheduled', - repository: { owner: o, name: n, fullName, defaultBranch: '' }, - repoPath, - }); - } catch (writeError) { - console.error(`::warning::Failed to write findings output: ${writeError}`); - } + const fullName = process.env['GITHUB_REPOSITORY'] ?? ''; + const [o = '', n = ''] = fullName.split('/'); + workflowSpan.setAttribute('warden.trigger.count', 0); + workflowSpan.setAttribute('warden.finding.count', 0); + const emptyContext: EventContext = { + eventType: 'schedule', + action: 'scheduled', + repository: { owner: o, name: n, fullName, defaultBranch: '' }, + repoPath, + }; + writeFindingsOutputs( + () => writeFindingsOutput([], emptyContext, [], { configuredSkills: [] }), + () => writeSchemaV2ScheduleOutputs(inputs, emptyContext, [], [], []), + (message) => console.error(`::warning::${message}`) + ); return; } throw error; @@ -129,18 +186,21 @@ async function runScheduleWorkflowInner( setOutput('high-count', 0); setOutput('summary', 'No schedule triggers configured'); workflowSpan.setAttribute('warden.finding.count', 0); - try { - const fullName = process.env['GITHUB_REPOSITORY'] ?? ''; - const [o = '', n = ''] = fullName.split('/'); - writeFindingsOutput([], { - eventType: 'schedule', - action: 'scheduled', - repository: { owner: o, name: n, fullName, defaultBranch: '' }, - repoPath, - }); - } catch (writeError) { - console.error(`::warning::Failed to write findings output: ${writeError}`); - } + const fullName = process.env['GITHUB_REPOSITORY'] ?? ''; + const [o = '', n = ''] = fullName.split('/'); + const emptyContext: EventContext = { + eventType: 'schedule', + action: 'scheduled', + repository: { owner: o, name: n, fullName, defaultBranch: '' }, + repoPath, + }; + writeFindingsOutputs( + () => writeFindingsOutput([], emptyContext, [], { + configuredSkills: buildConfiguredSkillsList({ allTriggers: allResolvedTriggers, matchedTriggers: [] }), + }), + () => writeSchemaV2ScheduleOutputs(inputs, emptyContext, allResolvedTriggers, [], []), + (message) => console.error(`::warning::${message}`) + ); return; } @@ -160,6 +220,14 @@ async function runScheduleWorkflowInner( const defaultBranch = await getDefaultBranchFromAPI(octokit, owner, repo); + // Hoisted above the trigger loop so the live-write path can use it as each trigger completes. + const scheduleContext: EventContext = { + eventType: 'schedule', + action: 'scheduled', + repository: { owner, name: repo, fullName: `${owner}/${repo}`, defaultBranch }, + repoPath, + }; + logGroup('Processing schedule triggers'); for (const trigger of scheduleTriggers) { console.log(`- ${trigger.name}: ${trigger.skill}`); @@ -167,6 +235,9 @@ async function runScheduleWorkflowInner( logGroupEnd(); const allReports: SkillReport[] = []; + const matchedTriggers: ResolvedTrigger[] = []; + const results: TriggerResult[] = []; + const processedTriggerIds = new Set(); let totalFindings = 0; const failureReasons: string[] = []; const triggerErrors: string[] = []; @@ -174,6 +245,7 @@ async function runScheduleWorkflowInner( // Process each schedule trigger for (const resolved of scheduleTriggers) { + processedTriggerIds.add(resolved.id); logGroup(`Running trigger: ${resolved.name} (skill: ${resolved.skill})`); try { @@ -210,6 +282,7 @@ async function runScheduleWorkflowInner( remote: resolved.remote, }); const runtimeEnv = await prepareRuntimeEnvironment([resolved], inputs); + const findingProcessingEvents: FindingProcessingEvent[] = []; const report = await runSkill(skill, context, { apiKey: inputs.anthropicApiKey, model: resolved.model, @@ -227,26 +300,52 @@ async function runScheduleWorkflowInner( verifyFindings: resolved.verifyFindings, triggerName: resolved.name, pathToClaudeCodeExecutable: runtimeEnv.pathToClaudeCodeExecutable, + callbacks: { + onFindingProcessing: (event) => findingProcessingEvents.push(event), + }, }); console.log(`Found ${report.findings.length} findings`); allReports.push(report); - totalFindings += report.findings.length; + matchedTriggers.push(resolved); - // Create/update issue with findings + // Create/update issue with findings. Failures here are logged but must + // not drop the trigger's own successful report/findings from `results` + // - that would desync the v2 output (which reads from `results`) from + // `allReports`/`totalFindings` (which already counted this report). const scheduleConfig: Partial = resolved.schedule ?? {}; const issueTitle = scheduleConfig.issueTitle ?? `Warden: ${resolved.name}`; - const issueResult = await createOrUpdateIssue(octokit, owner, repo, [report], { - title: issueTitle, - commitSha: headSha, - }); + let issueResult: IssueResult | null = null; + try { + issueResult = await createOrUpdateIssue(octokit, owner, repo, [report], { + title: issueTitle, + commitSha: headSha, + }); + } catch (error) { + console.error(`::warning::Failed to create/update issue for ${resolved.name}: ${error}`); + } if (issueResult) { console.log(`${issueResult.created ? 'Created' : 'Updated'} issue #${issueResult.issueNumber}`); console.log(`Issue URL: ${issueResult.issueUrl}`); } + results.push({ + triggerId: resolved.id, + triggerName: resolved.name, + skillName: resolved.skill, + skillExecutionId: resolved.skillExecutionId, + report, + findingProcessingEvents, + issueNumber: issueResult?.issueNumber, + issueUrl: issueResult?.issueUrl, + }); + writeSchemaV2ScheduleOutputsLive( + inputs, scheduleContext, allResolvedTriggers, matchedTriggers, results, processedTriggerIds + ); + totalFindings += report.findings.length; + // Check failure condition // Filter by confidence first so low-confidence findings don't cause failure const failOn = resolved.failOn ?? inputs.failOn; @@ -267,13 +366,22 @@ async function runScheduleWorkflowInner( }); const errorMessage = error instanceof Error ? error.message : String(error); triggerErrors.push(`${resolved.name}: ${errorMessage}`); + matchedTriggers.push(resolved); + results.push({ + triggerId: resolved.id, + triggerName: resolved.name, + skillName: resolved.skill, + skillExecutionId: resolved.skillExecutionId, + error, + }); + writeSchemaV2ScheduleOutputsLive( + inputs, scheduleContext, allResolvedTriggers, matchedTriggers, results, processedTriggerIds + ); console.error(`::warning::Trigger ${resolved.name} failed: ${error}`); logGroupEnd(); } } - handleTriggerErrors(triggerErrors, scheduleTriggers.length); - // Set outputs const highCount = countSeverity(allReports, 'high'); workflowSpan.setAttribute('warden.finding.count', totalFindings); @@ -283,17 +391,18 @@ async function runScheduleWorkflowInner( setOutput('summary', allReports.map((r) => r.summary).join('\n') || 'Scheduled analysis complete'); // Write structured findings to file for external export (GCS, S3, etc.) - try { - const findingsPath = writeFindingsOutput(allReports, { - eventType: 'schedule', - action: 'scheduled', - repository: { owner, name: repo, fullName: `${owner}/${repo}`, defaultBranch }, - repoPath, - }); - console.log(`Findings written to ${findingsPath}`); - } catch (error) { - console.error(`::warning::Failed to write findings output: ${error}`); - } + writeFindingsOutputs( + () => writeFindingsOutput(allReports, scheduleContext, [], { + configuredSkills: buildConfiguredSkillsList({ allTriggers: allResolvedTriggers, matchedTriggers }), + }), + () => writeSchemaV2ScheduleOutputs(inputs, scheduleContext, allResolvedTriggers, matchedTriggers, results), + (message) => console.error(`::warning::${message}`), + (path) => console.log(`Findings written to ${path}`) + ); + + // Both outputs are written above, so a total-failure or threshold exit below + // never discards artifacts that already reflect this run's results. + handleTriggerErrors(triggerErrors, scheduleTriggers.length); if (shouldFailAction) { setFailed(failureReasons.join('; ')); diff --git a/packages/warden/src/cli/commands/runs.test.ts b/packages/warden/src/cli/commands/runs.test.ts index 531601a8d..b132baf03 100644 --- a/packages/warden/src/cli/commands/runs.test.ts +++ b/packages/warden/src/cli/commands/runs.test.ts @@ -337,6 +337,186 @@ describe('runRunsShow', () => { }); }); +describe('runRunsShow — schema v2 replay', () => { + let testDir: string; + + beforeEach(() => { + testDir = join(tmpdir(), `warden-logs-show-v2-${Date.now()}`); + mkdirSync(testDir, { recursive: true }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true }); + } + }); + + function writeV2Fixture( + dir: string, + runId: string, + overrides: { metadata?: Record; findings?: Record } = {}, + ): { metadataPath: string; findingsPath: string } { + const metadata = { + schemaVersion: '2', + runId, + generatedAt: new Date('2026-02-18T10:00:00.000Z').toISOString(), + harness: { name: 'warden', version: '0.42.0' }, + repository: { owner: 'acme', name: 'repo', fullName: 'acme/repo' }, + event: 'pull_request', + ...overrides.metadata, + }; + + const findings = { + schemaVersion: '2', + runId, + skillExecutions: [ + { + skillExecutionId: 'exec-1', + skillName: 'security-review', + summary: 'Found 1 issue', + durationMs: 1500, + findingsBySeverity: { high: 1, medium: 0, low: 0 }, + findingIds: ['f1'], + }, + ], + findings: [ + { + id: 'f1', + contentHash: 'hash1', + severity: 'high', + title: 'SQL Injection', + description: 'Bad query', + reportedBy: [{ skillExecutionId: 'exec-1', skillName: 'security-review', role: 'primary' }], + provenance: { originSkillExecutionId: 'exec-1' }, + }, + ], + findingObservations: [], + summary: { + totalFindings: 1, + totalSkillExecutions: 1, + bySeverity: { high: 1, medium: 0, low: 0 }, + byOutcome: { posted: 1, deduped: 0, skipped: 0, resolved: 0, failed: 0 }, + }, + ...overrides.findings, + }; + + const metadataPath = join(dir, 'warden-metadata.json'); + const findingsPath = join(dir, 'warden-findings-v2.json'); + writeFileSync(metadataPath, JSON.stringify(metadata)); + writeFileSync(findingsPath, JSON.stringify(findings)); + return { metadataPath, findingsPath }; + } + + it('replays a schema-v2 metadata+findings pair', async () => { + const { metadataPath, findingsPath } = writeV2Fixture(testDir, 'run-1234'); + + const reporter = createTestReporter(); + const options = createDefaultOptions(); + + const exitCode = await runRunsShow( + { subcommand: 'show', files: [metadataPath, findingsPath] }, + options, + reporter, + ); + expect(exitCode).toBe(0); + }); + + it('warns that the run may still be in progress when no .done marker is present', async () => { + const { metadataPath, findingsPath } = writeV2Fixture(testDir, 'run-1234'); + + const reporter = createTestReporter(); + const warningSpy = vi.spyOn(reporter, 'warning'); + + const exitCode = await runRunsShow( + { subcommand: 'show', files: [metadataPath, findingsPath] }, + createDefaultOptions(), + reporter, + ); + + expect(exitCode).toBe(0); + expect(warningSpy).toHaveBeenCalledWith( + 'No .done marker found — this run may still be in progress; showing the latest snapshot' + ); + }); + + it('does not warn about an in-progress run when the .done marker is present', async () => { + const { metadataPath, findingsPath } = writeV2Fixture(testDir, 'run-1234'); + writeFileSync(`${findingsPath}.done`, ''); + + const reporter = createTestReporter(); + const warningSpy = vi.spyOn(reporter, 'warning'); + + const exitCode = await runRunsShow( + { subcommand: 'show', files: [metadataPath, findingsPath] }, + createDefaultOptions(), + reporter, + ); + + expect(exitCode).toBe(0); + expect(warningSpy).not.toHaveBeenCalled(); + }); + + it('renders the finding in JSON mode regardless of file order', async () => { + const { metadataPath, findingsPath } = writeV2Fixture(testDir, 'run-1234'); + + const reporter = createTestReporter(); + const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + + const exitCode = await runRunsShow( + { subcommand: 'show', files: [findingsPath, metadataPath] }, + { ...createDefaultOptions(), json: true }, + reporter, + ); + + expect(exitCode).toBe(0); + const output = stdoutSpy.mock.calls.map((call) => call[0]).join(''); + expect(output).toContain('SQL Injection'); + + stdoutSpy.mockRestore(); + }); + + it('errors when runId mismatches between metadata and findings', async () => { + const { metadataPath, findingsPath } = writeV2Fixture(testDir, 'run-1234', { + findings: { runId: 'run-9999' }, + }); + + const reporter = createTestReporter(); + const exitCode = await runRunsShow( + { subcommand: 'show', files: [metadataPath, findingsPath] }, + createDefaultOptions(), + reporter, + ); + expect(exitCode).toBe(1); + }); + + it('errors when only one file of the pair is given', async () => { + const { metadataPath } = writeV2Fixture(testDir, 'run-1234'); + + const reporter = createTestReporter(); + const exitCode = await runRunsShow( + { subcommand: 'show', files: [metadataPath] }, + createDefaultOptions(), + reporter, + ); + expect(exitCode).toBe(1); + }); + + it('errors when both files parse as the same schema-v2 kind', async () => { + const { metadataPath } = writeV2Fixture(testDir, 'run-1234'); + const secondMetadataPath = join(testDir, 'warden-metadata-2.json'); + writeFileSync(secondMetadataPath, readFileSync(metadataPath, 'utf-8')); + + const reporter = createTestReporter(); + const exitCode = await runRunsShow( + { subcommand: 'show', files: [metadataPath, secondMetadataPath] }, + createDefaultOptions(), + reporter, + ); + expect(exitCode).toBe(1); + }); +}); + describe('runRunsGc', () => { let testDir: string; @@ -1159,3 +1339,213 @@ describe('runRunsFollow', () => { stdoutSpy.mockRestore(); }, 5000); }); + +describe('runRunsFollowV2', () => { + let testDir: string; + + beforeEach(() => { + testDir = join(tmpdir(), `warden-follow-v2-${Date.now()}`); + mkdirSync(testDir, { recursive: true }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true }); + } + }); + + function writeV2FollowFixture( + runId: string, + skillExecutionIds: string[], + overrides: { metadata?: Record; findings?: Record } = {}, + ): { metadataPath: string; findingsPath: string } { + const metadata = { + schemaVersion: '2', + runId, + generatedAt: new Date('2026-02-18T10:00:00.000Z').toISOString(), + harness: { name: 'warden', version: '0.42.0' }, + repository: { owner: 'acme', name: 'repo', fullName: 'acme/repo' }, + event: 'pull_request', + ...overrides.metadata, + }; + + const findings = { + schemaVersion: '2', + runId, + skillExecutions: skillExecutionIds.map((id) => ({ + skillExecutionId: id, + skillName: `skill-${id}`, + summary: `Found 1 issue in ${id}`, + durationMs: 100, + findingsBySeverity: { high: 1, medium: 0, low: 0 }, + findingIds: [`f-${id}`], + })), + findings: skillExecutionIds.map((id) => ({ + id: `f-${id}`, + contentHash: `hash-${id}`, + severity: 'high', + title: `Finding from ${id}`, + description: 'test finding', + reportedBy: [{ skillExecutionId: id, skillName: `skill-${id}`, role: 'primary' }], + provenance: { originSkillExecutionId: id }, + })), + findingObservations: [], + summary: { + totalFindings: skillExecutionIds.length, + totalSkillExecutions: skillExecutionIds.length, + bySeverity: { high: skillExecutionIds.length, medium: 0, low: 0 }, + byOutcome: { posted: skillExecutionIds.length, deduped: 0, skipped: 0, resolved: 0, failed: 0 }, + }, + ...overrides.findings, + }; + + const metadataPath = join(testDir, 'warden-metadata.json'); + const findingsPath = join(testDir, 'warden-findings-v2.json'); + writeFileSync(metadataPath, JSON.stringify(metadata)); + writeFileSync(findingsPath, JSON.stringify(findings)); + return { metadataPath, findingsPath }; + } + + it('renders each newly-completed skill execution exactly once, then stops on .done', async () => { + const { metadataPath, findingsPath } = writeV2FollowFixture('run-live', ['exec-1']); + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const reporter = createTestReporter(); + const followPromise = runRunsFollow( + { subcommand: 'follow', files: [metadataPath, findingsPath] }, + createDefaultOptions(), + reporter, + ); + + await vi.waitFor(() => expect(logSpy).toHaveBeenCalledTimes(1), { timeout: 3000 }); + + writeV2FollowFixture('run-live', ['exec-1', 'exec-2']); + await vi.waitFor(() => expect(logSpy).toHaveBeenCalledTimes(2), { timeout: 3000 }); + + writeFileSync(`${findingsPath}.done`, ''); + const exit = await followPromise; + + expect(exit).toBe(0); + expect(logSpy).toHaveBeenCalledTimes(2); + }, 8000); + + it('with --json, emits one line per genuine change and suppresses an unchanged re-poll', async () => { + const { metadataPath, findingsPath } = writeV2FollowFixture('run-live', ['exec-1']); + + const writes: string[] = []; + const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + writes.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString('utf-8')); + return true; + }); + + const reporter = createTestReporter(); + const followPromise = runRunsFollow( + { subcommand: 'follow', files: [metadataPath, findingsPath] }, + { ...createDefaultOptions(), json: true }, + reporter, + ); + + await vi.waitFor(() => expect(writes.length).toBe(1), { timeout: 3000 }); + + // An unchanged re-poll (no rewrite at all) must not emit a duplicate line. + await new Promise((r) => setTimeout(r, 1200)); + expect(writes.length).toBe(1); + + writeV2FollowFixture('run-live', ['exec-1', 'exec-2']); + await vi.waitFor(() => expect(writes.length).toBe(2), { timeout: 3000 }); + + writeFileSync(`${findingsPath}.done`, ''); + const exit = await followPromise; + + expect(exit).toBe(0); + expect(writes.length).toBe(2); + expect(JSON.parse(writes[0]!).findings.skillExecutions).toHaveLength(1); + expect(JSON.parse(writes[1]!).findings.skillExecutions).toHaveLength(2); + + stdoutSpy.mockRestore(); + }, 8000); + + it('errors when given a file count other than exactly 2', async () => { + const { metadataPath, findingsPath } = writeV2FollowFixture('run-live', ['exec-1']); + const thirdPath = join(testDir, 'extra.json'); + writeFileSync(thirdPath, '{}'); + + const reporter = createTestReporter(); + const exit = await runRunsFollow( + { subcommand: 'follow', files: [metadataPath, findingsPath, thirdPath] }, + createDefaultOptions(), + reporter, + ); + + expect(exit).toBe(1); + }); + + it('errors when runId mismatches between metadata and findings, without ever rendering', async () => { + const { metadataPath, findingsPath } = writeV2FollowFixture('run-a', ['exec-1'], { + findings: { runId: 'run-b' }, + }); + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const reporter = createTestReporter(); + const followPromise = runRunsFollow( + { subcommand: 'follow', files: [metadataPath, findingsPath] }, + createDefaultOptions(), + reporter, + ); + + await new Promise((r) => setTimeout(r, 100)); + writeFileSync(`${findingsPath}.done`, ''); + const exit = await followPromise; + + expect(exit).toBe(0); + expect(logSpy).not.toHaveBeenCalled(); + }, 5000); + + it('tolerates a transient malformed file mid-tick and recovers on the next one', async () => { + const { metadataPath, findingsPath } = writeV2FollowFixture('run-live', ['exec-1']); + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const reporter = createTestReporter(); + const followPromise = runRunsFollow( + { subcommand: 'follow', files: [metadataPath, findingsPath] }, + createDefaultOptions(), + reporter, + ); + + await vi.waitFor(() => expect(logSpy).toHaveBeenCalledTimes(1), { timeout: 3000 }); + + // Simulate a reader catching the findings file mid-rewrite. + writeFileSync(findingsPath, '{"not valid json'); + await new Promise((r) => setTimeout(r, 50)); + writeV2FollowFixture('run-live', ['exec-1', 'exec-2']); + + writeFileSync(`${findingsPath}.done`, ''); + const exit = await followPromise; + + expect(exit).toBe(0); + expect(logSpy).toHaveBeenCalledTimes(2); + }, 8000); + + it('warns instead of silently exiting when .done lands but the pair still will not parse', async () => { + const { metadataPath, findingsPath } = writeV2FollowFixture('run-live', ['exec-1']); + vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const reporter = createTestReporter(); + const warningSpy = vi.spyOn(reporter, 'warning'); + const followPromise = runRunsFollow( + { subcommand: 'follow', files: [metadataPath, findingsPath] }, + createDefaultOptions(), + reporter, + ); + + await new Promise((r) => setTimeout(r, 50)); + writeFileSync(findingsPath, '{"still not valid json'); + writeFileSync(`${findingsPath}.done`, ''); + + const exit = await followPromise; + + expect(exit).toBe(0); + expect(warningSpy).toHaveBeenCalledWith('Run finished but its output could not be read'); + }, 8000); +}); diff --git a/packages/warden/src/cli/commands/runs.ts b/packages/warden/src/cli/commands/runs.ts index 6414a8e37..4ec7fa3fb 100644 --- a/packages/warden/src/cli/commands/runs.ts +++ b/packages/warden/src/cli/commands/runs.ts @@ -24,6 +24,13 @@ import { type JsonlSummaryRecord, type LogFileMetadata, } from '../output/index.js'; +import { + WardenMetadataSchema, + WardenFindingsSchemaV2, + reconstructSkillReportsFromV2, + type WardenMetadata, + type WardenFindingsV2, +} from '../../action/reporting/output-v2.js'; /** * Resolve a log directory path from the repo root. @@ -434,6 +441,10 @@ export async function runRunsShow( return 1; } + if (resolvedFiles.every((file) => !file.endsWith('.jsonl'))) { + return runRunsShowV2(resolvedFiles, options, reporter); + } + // Parse and merge reports from all files const allReports: SkillReport[] = []; let totalDurationMs = 0; @@ -503,6 +514,262 @@ export async function runRunsShow( return 0; } +/** Try to parse a file as one half of a schema-v2 metadata/findings artifact pair. */ +function parseV2File( + filePath: string, +): { kind: 'metadata'; data: WardenMetadata } | { kind: 'findings'; data: WardenFindingsV2 } | { kind: 'invalid' } { + let json: unknown; + try { + json = JSON.parse(readFileSync(filePath, 'utf-8')); + } catch { + return { kind: 'invalid' }; + } + + const metadataResult = WardenMetadataSchema.safeParse(json); + if (metadataResult.success) return { kind: 'metadata', data: metadataResult.data }; + + const findingsResult = WardenFindingsSchemaV2.safeParse(json); + if (findingsResult.success) return { kind: 'findings', data: findingsResult.data }; + + return { kind: 'invalid' }; +} + +/** + * Show results from a schema-v2 metadata+findings artifact pair (the GitHub + * Action's `output-schema-version: '2'` output) — the same after-the-fact + * replay `runRunsShow` gives v1 JSONL logs, reconstructed via + * `reconstructSkillReportsFromV2`. + */ +async function runRunsShowV2( + files: string[], + options: CLIOptions, + reporter: Reporter, +): Promise { + if (files.length !== 2) { + reporter.error(`Schema-v2 replay needs exactly one metadata file and one findings file, got ${files.length}`); + reporter.tip('Usage: warden runs show '); + return 1; + } + + let metadata: WardenMetadata | undefined; + let findings: WardenFindingsV2 | undefined; + let findingsFile: string | undefined; + + for (const file of files) { + const parsed = parseV2File(file); + if (parsed.kind === 'metadata') { + if (metadata) { + reporter.error(`Both files look like a schema-v2 metadata file: ${file}`); + return 1; + } + metadata = parsed.data; + } else if (parsed.kind === 'findings') { + if (findings) { + reporter.error(`Both files look like a schema-v2 findings file: ${file}`); + return 1; + } + findings = parsed.data; + findingsFile = file; + } else { + reporter.error(`${file} is not a recognized JSONL run log or schema-v2 metadata/findings file`); + return 1; + } + } + + if (!metadata || !findings || !findingsFile) { + reporter.error('Schema-v2 replay needs one metadata file and one findings file'); + return 1; + } + + if (!existsSync(`${findingsFile}.done`)) { + reporter.warning('No .done marker found — this run may still be in progress; showing the latest snapshot'); + } + + if (metadata.runId !== findings.runId || metadata.schemaVersion !== findings.schemaVersion) { + reporter.error('Metadata file and findings file do not share the same runId/schemaVersion'); + return 1; + } + + const reports = reconstructSkillReportsFromV2(findings); + if (reports.length === 0) { + reporter.warning('No skill executions found in findings file'); + return 0; + } + + const resolved = resolveLogDir(); + let configMinConfidence: ConfidenceThreshold | undefined; + if (resolved) { + try { + const configPath = resolve(resolved.repoPath, 'warden.toml'); + if (existsSync(configPath)) { + const config = loadWardenConfigFile(configPath); + configMinConfidence = config.defaults?.minConfidence; + } + } catch { + // Use default + } + } + + const totalDurationMs = reports.reduce((sum, report) => sum + (report.durationMs ?? 0), 0); + const filteredReports = filterReports(reports, options.reportOn, options.minConfidence ?? configMinConfidence ?? 'medium'); + + reporter.blank(); + if (options.json) { + const jsonlContent = renderJsonlString(filteredReports, totalDurationMs, { + runId: metadata.runId, + timestamp: new Date(metadata.generatedAt), + model: metadata.resolvedDefaults?.model, + headSha: metadata.pullRequest?.headSha, + }); + process.stdout.write(jsonlContent); + } else { + console.log(renderTerminalReport(filteredReports, reporter.mode, { verbosity: reporter.verbosity })); + } + + reporter.blank(); + reporter.renderSummary(filteredReports, totalDurationMs); + + return 0; +} + +/** Best-effort re-parse of both v2 files; returns undefined on any transient mismatch (in-flight write, race between the pair). */ +function tryReadV2Pair( + files: string[], +): { metadata: WardenMetadata; findings: WardenFindingsV2 } | undefined { + let metadata: WardenMetadata | undefined; + let findings: WardenFindingsV2 | undefined; + + for (const file of files) { + const parsed = parseV2File(file); + if (parsed.kind === 'metadata') { + if (metadata) return undefined; + metadata = parsed.data; + } else if (parsed.kind === 'findings') { + if (findings) return undefined; + findings = parsed.data; + } else { + return undefined; + } + } + + if (!metadata || !findings) return undefined; + if (metadata.runId !== findings.runId || metadata.schemaVersion !== findings.schemaVersion) return undefined; + return { metadata, findings }; +} + +/** + * Tail a schema-v2 metadata+findings pair, rendering each newly-completed + * skill execution as it lands. Unlike v1's JSONL follow, both files are + * fully rewritten on every update (no append semantics) — each tick + * re-reads and re-parses both from scratch, and a parse mismatch (one file + * updated, the other not yet) is a transient race to wait out, not an error. + * Stops once a `.done` sidecar appears next to either file, or on Ctrl-C. + */ +async function runRunsFollowV2( + files: string[], + options: CLIOptions, + reporter: Reporter, +): Promise { + if (files.length !== 2) { + reporter.error(`Schema-v2 follow needs exactly one metadata file and one findings file, got ${files.length}`); + reporter.tip('Usage: warden runs follow '); + return 1; + } + + const missingFiles = files.filter((file) => !existsSync(file)); + if (missingFiles.length > 0) { + reporter.error(`Log ${pluralize(missingFiles.length, 'file')} not found: ${missingFiles.join(', ')}`); + return 1; + } + + if (!options.json) { + reporter.dim(`Following: ${files.join(', ')}`); + reporter.blank(); + } + + const renderedIds = new Set(); + let lastEmittedRaw = ''; + let stopped = false; + + const renderNew = (findings: WardenFindingsV2): void => { + const reports = reconstructSkillReportsFromV2(findings); + findings.skillExecutions.forEach((execution, i) => { + if (renderedIds.has(execution.skillExecutionId)) return; + renderedIds.add(execution.skillExecutionId); + const report = reports[i]; + if (!report) return; + console.log(renderTerminalReport([report], reporter.mode, { verbosity: reporter.verbosity })); + }); + }; + + const emitJson = (metadata: WardenMetadata, findings: WardenFindingsV2): void => { + const raw = JSON.stringify({ metadata, findings }); + if (raw === lastEmittedRaw) return; + lastEmittedRaw = raw; + process.stdout.write(raw + '\n'); + }; + + const isDone = (): boolean => files.some((file) => existsSync(`${file}.done`)); + + const poll = (): void => { + const done = isDone(); + const result = tryReadV2Pair(files); + if (result) { + if (options.json) emitJson(result.metadata, result.findings); + else renderNew(result.findings); + } + if (!done) return; + + if (!options.json) { + const finalResult = result ?? tryReadV2Pair(files); + if (finalResult) { + reporter.blank(); + const reports = reconstructSkillReportsFromV2(finalResult.findings); + const totalDurationMs = reports.reduce((sum, r) => sum + (r.durationMs ?? 0), 0); + reporter.renderSummary(reports, totalDurationMs); + } else { + reporter.warning('Run finished but its output could not be read'); + } + } + stopped = true; + }; + + poll(); + if (stopped) return 0; + + return new Promise((resolvePromise) => { + const watchers: ReturnType[] = []; + + const tick = () => { + poll(); + if (stopped) finish(0); + }; + + try { + for (const file of files) { + const watcher = watch(file, { persistent: true }, () => tick()); + watcher.on('error', () => { + try { watcher.close(); } catch { /* ignore */ } + }); + watchers.push(watcher); + } + } catch { /* polling alone is fine */ } + const pollTimer = setInterval(tick, 1000); + + const finish = (code: number) => { + for (const watcher of watchers) { + try { watcher.close(); } catch { /* ignore */ } + } + clearInterval(pollTimer); + process.off('SIGINT', onSigint); + resolvePromise(code); + }; + + const onSigint = () => finish(0); + process.on('SIGINT', onSigint); + }); +} + /** * Garbage-collect expired log files. */ @@ -691,6 +958,14 @@ export async function runRunsFollow( options: CLIOptions, reporter: Reporter, ): Promise { + // Existing v1 follow only ever takes 0 (auto-detect) or 1 (run id / path) + // args; v2 has no auto-discovery, so more than one explicit file path is an + // unambiguous signal to follow a schema-v2 metadata+findings pair instead. + // runRunsFollowV2 itself validates the count is exactly 2. + if (runsOptions.files.length > 1) { + return runRunsFollowV2(runsOptions.files, options, reporter); + } + const resolved = resolveLogDir(); if (!resolved) { reporter.error('Not a git repository'); diff --git a/packages/warden/src/cli/help.ts b/packages/warden/src/cli/help.ts index c90b7f1b5..3059e58a1 100644 --- a/packages/warden/src/cli/help.ts +++ b/packages/warden/src/cli/help.ts @@ -406,31 +406,34 @@ const HELP_COMMANDS: Record = { }, 'runs:show': { summary: 'Show results from saved logs or run IDs', - description: 'Render findings from one or more saved JSONL logs or short run IDs.', + description: 'Render findings from one or more saved JSONL logs, short run IDs, or a schema-v2 metadata+findings file pair from the GitHub Action.', usage: ['warden runs show [options]'], arguments: [ - { label: 'files...', description: 'JSONL paths or short run IDs' }, + { label: 'files...', description: 'JSONL paths, short run IDs, or a warden-metadata.json + warden-findings-v2.json pair' }, ], options: ['cwd', 'json', 'reportOn', 'minConfidence', ...SHARED_COMMAND_OPTIONS], examples: [ 'warden runs show deadbeef', 'warden runs show .warden/logs/a1b2c3d4-2026-04-25T13-00-00-000Z.jsonl', + 'warden runs show warden-metadata.json warden-findings-v2.json', ], }, 'runs:follow': { summary: 'Tail a live or completed session', - description: 'Follow a session as skill records arrive, or replay the latest completed run.', + description: 'Follow a session as skill records arrive, or replay the latest completed run. Pass a schema-v2 metadata+findings file pair to follow a local GitHub Action workflow run instead.', usage: [ 'warden runs follow [run] [options]', 'warden runs --follow [run] [options]', + 'warden runs follow [options]', ], arguments: [ - { label: 'run', description: 'Optional short run ID or JSONL path' }, + { label: 'run', description: 'Optional short run ID or JSONL path, or a warden-metadata.json + warden-findings-v2.json pair' }, ], options: ['cwd', 'json', ...SHARED_COMMAND_OPTIONS], examples: [ 'warden runs follow', 'warden runs --follow deadbeef', + 'warden runs follow warden-metadata.json warden-findings-v2.json', ], }, 'runs:gc': { diff --git a/packages/warden/src/cli/main.test.ts b/packages/warden/src/cli/main.test.ts index 0b7322c75..8c6f5891e 100644 --- a/packages/warden/src/cli/main.test.ts +++ b/packages/warden/src/cli/main.test.ts @@ -284,6 +284,88 @@ describe('buildFinalChunkRecords', () => { expect(summary.usageBreakdown?.auxiliary?.['verification']?.model).toBe('verify-model'); expect(summary.usageBreakdown?.total.usage.costUSD).toBeCloseTo(6.5); }); + + it('survives verifierRejections through the finalized chunk stream and reconstruction', () => { + const run = buildRunMetadata({ + runId: 'run-verifier-rejections', + durationMs: 100, + timestamp: new Date('2026-06-03T00:00:00.000Z'), + cwd: '/repo', + }); + const chunk: JsonlChunkRecord = { + schemaVersion: 1, + run, + skill: 'skill-a', + model: 'scan-model', + chunk: { file: 'src/app.ts', index: 1, total: 1, lineRange: '10-20' }, + status: 'ok', + findings: [], + usageBreakdown: buildJsonlUsageBreakdown(makeUsage(1000, 100, 5), undefined), + durationMs: 100, + }; + const log: RunLog = { + paths: [], + primaryLogPath: '/repo/.warden/logs/run.jsonl', + primaryLogWritten: true, + outputPath: undefined, + startTime: 0, + baseRun: run, + chunks: [chunk], + }; + const report = makeReport({ + model: 'scan-model', + usage: makeUsage(1000, 100, 5), + verifierRejections: { count: 2, reasons: ['not reproducible', 'guarded upstream'] }, + }); + + const records = buildFinalChunkRecords(log, [report], 1000); + const postProcessing = records.find((record) => record.chunk.lineRange === 'post-processing'); + expect(postProcessing?.verifierRejections).toEqual({ + count: 2, + reasons: ['not reproducible', 'guarded upstream'], + }); + + const parsed = parseJsonlReports(renderJsonlChunkRecords(records)); + expect(parsed.reports).toHaveLength(1); + expect(parsed.reports[0]!.verifierRejections).toEqual({ + count: 2, + reasons: ['not reproducible', 'guarded upstream'], + }); + }); + + it('omits a post-processing record when there is no missing usage or verifier rejections', () => { + const run = buildRunMetadata({ + runId: 'run-no-post-processing', + durationMs: 100, + timestamp: new Date('2026-06-03T00:00:00.000Z'), + cwd: '/repo', + }); + const chunk: JsonlChunkRecord = { + schemaVersion: 1, + run, + skill: 'skill-a', + model: 'scan-model', + chunk: { file: 'src/app.ts', index: 1, total: 1, lineRange: '10-20' }, + status: 'ok', + findings: [], + usageBreakdown: buildJsonlUsageBreakdown(makeUsage(1000, 100, 5), undefined), + durationMs: 100, + }; + const log: RunLog = { + paths: [], + primaryLogPath: '/repo/.warden/logs/run.jsonl', + primaryLogWritten: true, + outputPath: undefined, + startTime: 0, + baseRun: run, + chunks: [chunk], + }; + const report = makeReport({ model: 'scan-model', usage: makeUsage(1000, 100, 5) }); + + const records = buildFinalChunkRecords(log, [report], 1000); + + expect(records).toHaveLength(1); + }); }); describe('appendReportToRunLog', () => { diff --git a/packages/warden/src/cli/main.ts b/packages/warden/src/cli/main.ts index e3f3135a2..2ecd0abbb 100644 --- a/packages/warden/src/cli/main.ts +++ b/packages/warden/src/cli/main.ts @@ -14,7 +14,7 @@ import { mapExtractionErrorCode } from '../sdk/errors.js'; import { aggregateAuxiliaryUsageAttribution, mergeAuxiliaryUsage } from '../sdk/usage.js'; import { resolveSkillAsync, SkillLoaderError } from '../skills/loader.js'; import { matchTrigger, filterContextByPaths, shouldFail, countFindingsAtOrAbove } from '../triggers/matcher.js'; -import type { SkillReport, SeverityThreshold, ConfidenceThreshold, SkillError, Finding, UsageStats, AuxiliaryUsageMap } from '../types/index.js'; +import type { SkillReport, SeverityThreshold, ConfidenceThreshold, SkillError, Finding, UsageStats, AuxiliaryUsageMap, VerifierRejections } from '../types/index.js'; import { filterFindings } from '../types/index.js'; import { DEFAULT_CONCURRENCY, getAnthropicApiKey } from '../utils/index.js'; import { isRepoRelativePath, normalizePath, resolveConfigInput } from '../utils/path.js'; @@ -333,6 +333,7 @@ function buildReportChunkRecord( durationMs: includeReportResults ? (report.durationMs ?? runDurationMs) : 0, error: reportError, skippedFiles: report.skippedFiles, + verifierRejections: includeReportResults ? report.verifierRejections : undefined, }; } @@ -470,7 +471,8 @@ function buildUsageOnlyChunkRecord( log: RunLog, report: SkillReport, runDurationMs: number, - auxiliaryUsage: AuxiliaryUsageMap, + auxiliaryUsage: AuxiliaryUsageMap | undefined, + verifierRejections: VerifierRejections | undefined, ): JsonlChunkRecord { return { schemaVersion: 1, @@ -489,6 +491,7 @@ function buildUsageOnlyChunkRecord( usageBreakdown: buildJsonlUsageBreakdown(undefined, auxiliaryUsage, { auxiliary: report.auxiliaryUsageAttribution, }), + verifierRejections, }; } @@ -538,8 +541,8 @@ export function buildFinalChunkRecords( undefined, ); const missingAuxiliaryUsage = subtractAuxiliaryUsage(report.auxiliaryUsage, recordedAuxiliaryUsage); - return missingAuxiliaryUsage - ? [buildUsageOnlyChunkRecord(log, report, totalDurationMs, missingAuxiliaryUsage)] + return missingAuxiliaryUsage || report.verifierRejections + ? [buildUsageOnlyChunkRecord(log, report, totalDurationMs, missingAuxiliaryUsage, report.verifierRejections)] : []; }); return [ diff --git a/packages/warden/src/cli/output/formatters.ts b/packages/warden/src/cli/output/formatters.ts index 99e385f26..6529413ae 100644 --- a/packages/warden/src/cli/output/formatters.ts +++ b/packages/warden/src/cli/output/formatters.ts @@ -2,6 +2,11 @@ import chalk from 'chalk'; import figures from 'figures'; import type { Severity, Confidence, Finding, FileChange, UsageStats, AuxiliaryUsageMap } from '../../types/index.js'; +/** The id to display for a finding: its reportedId once dedupe sets one, otherwise its own id. */ +export function displayFindingId(finding: Finding): string { + return finding.reportedId ?? finding.id; +} + /** * Capitalize the first letter of a string. * @example capitalize('critical') // 'Critical' @@ -142,7 +147,7 @@ export function formatLocation(path: string, startLine?: number, endLine?: numbe */ export function formatFindingCompact(finding: Finding): string { const badge = formatSeverityBadge(finding.severity); - const id = chalk.dim(`[${finding.id}]`); + const id = chalk.dim(`[${displayFindingId(finding)}]`); const location = finding.location ? chalk.dim(formatLocation(finding.location.path, finding.location.startLine, finding.location.endLine)) : ''; diff --git a/packages/warden/src/cli/output/jsonl.test.ts b/packages/warden/src/cli/output/jsonl.test.ts index 886686251..916efdb29 100644 --- a/packages/warden/src/cli/output/jsonl.test.ts +++ b/packages/warden/src/cli/output/jsonl.test.ts @@ -908,6 +908,45 @@ describe('parseJsonlReports', () => { expect(report.auxiliaryUsage?.['verification']?.costUSD).toBe(0.0005); }); + it('recovers verifierRejections from the post-processing chunk marker', () => { + const run = buildRunMetadata({ + runId: 'chunk-verifier-rejections', + durationMs: 100, + timestamp: new Date('2026-02-18T14:32:15.123Z'), + cwd: '/test', + }); + const chunks: JsonlChunkRecord[] = [ + { + schemaVersion: 1, + run, + skill: 'security-review', + chunk: { file: 'src/api.ts', index: 1, total: 1, lineRange: '10-20' }, + status: 'ok', + findings: [], + usageBreakdown: buildJsonlUsageBreakdown({ inputTokens: 100, outputTokens: 10, costUSD: 0.001 }, undefined), + durationMs: 100, + }, + { + schemaVersion: 1, + run, + skill: 'security-review', + chunk: { file: '', index: 1, total: 1, lineRange: 'post-processing' }, + status: 'ok', + findings: [], + durationMs: 0, + verifierRejections: { count: 2, reasons: ['not reproducible', 'guarded upstream'] }, + }, + ]; + + const result = parseJsonlReports(chunks.map((chunk) => renderJsonlChunkLine(chunk)).join('')); + + expect(result.reports).toHaveLength(1); + expect(result.reports[0]!.verifierRejections).toEqual({ + count: 2, + reasons: ['not reproducible', 'guarded upstream'], + }); + }); + it('reconstructs trace metadata from chunk records', () => { const run = buildRunMetadata({ runId: 'chunk-traces', durationMs: 300 }); const trace: HunkTrace = { @@ -1228,6 +1267,7 @@ another bad line summary: 'ok', findings: [], failedExtractions: 1, + verifierRejections: { count: 3, reasons: ['a', 'b', 'c'] }, }, ]; const content = renderJsonlString(original, 1000, { runId: 'sum-123' }); @@ -1237,6 +1277,7 @@ another bad line expect(summary.failedSkills).toEqual(['skill-a']); expect(summary.totalFailedHunks).toBe(2); expect(summary.totalFailedExtractions).toBe(1); + expect(summary.totalVerifierRejections).toBe(3); }); it('omits summary extras when there are no failures', () => { @@ -1249,6 +1290,7 @@ another bad line expect(summary.failedSkills).toBeUndefined(); expect(summary.totalFailedHunks).toBeUndefined(); expect(summary.totalFailedExtractions).toBeUndefined(); + expect(summary.totalVerifierRejections).toBeUndefined(); }); it('passes through fix-evaluation records without treating them as malformed', () => { diff --git a/packages/warden/src/cli/output/jsonl.ts b/packages/warden/src/cli/output/jsonl.ts index 9f87ff990..91300eefa 100644 --- a/packages/warden/src/cli/output/jsonl.ts +++ b/packages/warden/src/cli/output/jsonl.ts @@ -13,6 +13,7 @@ import { SkillErrorSchema, SkippedFileSchema, HunkTraceSchema, + VerifierRejectionsSchema, } from '../../types/index.js'; import type { SkillReport, UsageStats, AuxiliaryUsageMap, AuxiliaryUsageAttributionMap, SkillError, Finding, FileReport, HunkFailure } from '../../types/index.js'; import { mergeAuxiliaryUsage, mergeAuxiliaryUsageAttribution } from '../../sdk/usage.js'; @@ -120,6 +121,7 @@ export const JsonlChunkRecordSchema = z.object({ error: SkillErrorSchema.optional(), skippedFiles: z.array(SkippedFileSchema).optional(), trace: HunkTraceSchema.optional(), + verifierRejections: VerifierRejectionsSchema.optional(), }); export type JsonlChunkRecord = z.infer; @@ -200,6 +202,7 @@ export const JsonlSummaryRecordSchema = z.object({ failedSkills: z.array(z.string()).optional(), totalFailedHunks: z.number().int().nonnegative().optional(), totalFailedExtractions: z.number().int().nonnegative().optional(), + totalVerifierRejections: z.number().int().nonnegative().optional(), /** * Top-level run error captured before any skill ran (e.g. auth failure, * config load error). Skill-level errors live on the SkillRecord; this @@ -330,6 +333,7 @@ export function buildSummaryJsonlRecord( const failedSkills = reports.filter((r) => r.error).map((r) => r.skill); const totalFailedHunks = reports.reduce((n, r) => n + (r.failedHunks ?? 0), 0); const totalFailedExtractions = reports.reduce((n, r) => n + (r.failedExtractions ?? 0), 0); + const totalVerifierRejections = reports.reduce((n, r) => n + (r.verifierRejections?.count ?? 0), 0); return { run, type: 'summary', @@ -343,6 +347,7 @@ export function buildSummaryJsonlRecord( failedSkills: failedSkills.length > 0 ? failedSkills : undefined, totalFailedHunks: totalFailedHunks > 0 ? totalFailedHunks : undefined, totalFailedExtractions: totalFailedExtractions > 0 ? totalFailedExtractions : undefined, + totalVerifierRejections: totalVerifierRejections > 0 ? totalVerifierRejections : undefined, error, }; } @@ -551,6 +556,7 @@ function reportsFromChunks(chunks: JsonlChunkRecord[]): SkillReport[] { const failedExtractions = analysisChunkRecords.filter( (r) => r.status === 'error' && r.error && isExtractionErrorCode(r.error.code), ).length; + const verifierRejections = aggregateRecords.find((r) => r.verifierRejections)?.verifierRejections; const allChunksFailed = analysisChunkRecords.length > 0 && findings.length === 0 && @@ -576,6 +582,7 @@ function reportsFromChunks(chunks: JsonlChunkRecord[]): SkillReport[] { if (auxiliaryUsageAttribution) report.auxiliaryUsageAttribution = auxiliaryUsageAttribution; if (failedHunks > 0) report.failedHunks = failedHunks; if (failedExtractions > 0) report.failedExtractions = failedExtractions; + if (verifierRejections) report.verifierRejections = verifierRejections; if (hunkFailures.length > 0) report.hunkFailures = hunkFailures; if (traces.length > 0) report.traces = traces; if (skippedFiles.length > 0) report.skippedFiles = skippedFiles; diff --git a/packages/warden/src/cli/output/reporter.test.ts b/packages/warden/src/cli/output/reporter.test.ts index b884406ac..37a551965 100644 --- a/packages/warden/src/cli/output/reporter.test.ts +++ b/packages/warden/src/cli/output/reporter.test.ts @@ -191,5 +191,32 @@ describe('Reporter', () => { expect(errorOutput).not.toContain('failed to analyze'); expect(errorOutput).not.toContain('Use -v'); }); + + it('shows verifier rejection count in log mode', () => { + const reporter = new Reporter(logMode(), Verbosity.Normal); + reporter.renderSummary([makeReport({ verifierRejections: { count: 4, reasons: ['a', 'b', 'c', 'd'] } })], 1000); + + const output = errorSpy.mock.calls.map((c: unknown[]) => c[0] as string).join('\n'); + expect(output).toContain('WARN: 4 findings rejected by verification'); + }); + + it('shows verifier rejection count in TTY mode', () => { + const reporter = new Reporter(ttyMode(), Verbosity.Normal); + reporter.renderSummary([makeReport({ verifierRejections: { count: 1, reasons: ['a'] } })], 1000); + + const output = errorSpy.mock.calls.map((c: unknown[]) => c[0] as string).join('\n'); + expect(output).toContain('1 finding rejected by verification'); + }); + + it('aggregates verifier rejections across multiple reports', () => { + const reporter = new Reporter(logMode(), Verbosity.Normal); + reporter.renderSummary([ + makeReport({ verifierRejections: { count: 1, reasons: ['a'] } }), + makeReport({ verifierRejections: { count: 2, reasons: ['b', 'c'] } }), + ], 1000); + + const output = errorSpy.mock.calls.map((c: unknown[]) => c[0] as string).join('\n'); + expect(output).toContain('WARN: 3 findings rejected by verification'); + }); }); }); diff --git a/packages/warden/src/cli/output/reporter.ts b/packages/warden/src/cli/output/reporter.ts index 3d29157de..42de82a19 100644 --- a/packages/warden/src/cli/output/reporter.ts +++ b/packages/warden/src/cli/output/reporter.ts @@ -207,11 +207,13 @@ export class Reporter { let totalFailedHunks = 0; let totalFailedExtractions = 0; let totalSkippedFiles = 0; + let totalVerifierRejections = 0; for (const report of reports) { allFindings.push(...report.findings); totalFailedHunks += report.failedHunks ?? 0; totalFailedExtractions += report.failedExtractions ?? 0; totalSkippedFiles += report.skippedFiles?.length ?? 0; + totalVerifierRejections += report.verifierRejections?.count ?? 0; } const counts = countBySeverity(allFindings); const totalUsage = this.aggregateUsage(reports); @@ -233,6 +235,9 @@ export class Reporter { if (totalFailedExtractions > 0) { this.log(chalk.yellow(`${figures.warning} ${totalFailedExtractions} finding ${pluralize(totalFailedExtractions, 'extraction')} failed`)); } + if (totalVerifierRejections > 0) { + this.log(chalk.yellow(`${figures.warning} ${totalVerifierRejections} ${pluralize(totalVerifierRejections, 'finding')} rejected by verification`)); + } if ((totalFailedHunks > 0 || totalFailedExtractions > 0) && this.verbosity < Verbosity.Verbose) { this.log(chalk.dim(' Use -v for failure details')); } @@ -256,6 +261,9 @@ export class Reporter { if (totalFailedExtractions > 0) { this.logPlain(`WARN: ${totalFailedExtractions} finding ${pluralize(totalFailedExtractions, 'extraction')} failed`); } + if (totalVerifierRejections > 0) { + this.logPlain(`WARN: ${totalVerifierRejections} ${pluralize(totalVerifierRejections, 'finding')} rejected by verification`); + } if ((totalFailedHunks > 0 || totalFailedExtractions > 0) && this.verbosity < Verbosity.Verbose) { this.logPlain('Use -v for failure details'); } diff --git a/packages/warden/src/cli/output/tasks.test.ts b/packages/warden/src/cli/output/tasks.test.ts index edc9a3971..45fe6450b 100644 --- a/packages/warden/src/cli/output/tasks.test.ts +++ b/packages/warden/src/cli/output/tasks.test.ts @@ -682,6 +682,7 @@ describe('runSkillTasks', () => { const postProcessFindings = vi.spyOn(sdkRunner, 'postProcessFindings').mockResolvedValue({ findings: [], auxiliaryUsage: [], + verifierRejections: { count: 1, reasons: ['not reproducible'] }, }); const results = await runSkillTasks([{ @@ -705,6 +706,55 @@ describe('runSkillTasks', () => { expect(results[0]?.report?.files?.[0]?.findings).toBe(0); expect(controller.signal.aborted).toBe(false); expect(postProcessFindings).toHaveBeenCalled(); + expect(results[0]?.report?.verifierRejections).toEqual({ count: 1, reasons: ['not reproducible'] }); + + prepareFiles.mockRestore(); + analyzeFile.mockRestore(); + postProcessFindings.mockRestore(); + }); + + it('omits verifierRejections on the report when nothing is rejected', async () => { + const finalFinding = makeFinding(); + const controller = new AbortController(); + const fakeHunk = { + hunk: { newStart: 1, newCount: 10 }, + } as unknown as HunkWithContext; + + const prepareFiles = vi.spyOn(sdkRunner, 'prepareFiles').mockReturnValue({ + files: [{ filename: 'a.ts', hunks: [fakeHunk] }], + skippedFiles: [], + }); + const analyzeFile = vi.spyOn(sdkRunner, 'analyzeFile').mockResolvedValue({ + filename: 'a.ts', + findings: [finalFinding], + usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.001 }, + failedHunks: 0, + failedExtractions: 0, + hunkFailures: [], + } satisfies FileAnalysisResult); + const postProcessFindings = vi.spyOn(sdkRunner, 'postProcessFindings').mockResolvedValue({ + findings: [finalFinding], + auxiliaryUsage: [], + }); + + const results = await runSkillTasks([{ + name: 'verify-skill', + resolveSkill: async () => + ({ name: 'verify-skill', definition: '', files: [] } as unknown as SkillDefinition), + context: { + eventType: 'pull_request', + repository: { owner: 'o', name: 'n', fullName: 'o/n', defaultBranch: 'main' }, + repoPath: '/tmp', + pullRequest: { number: 1, title: 't', body: '', headSha: 'abc', baseSha: 'def', files: [] }, + } as unknown as SkillTaskOptions['context'], + }], { + mode: logMode(), + verbosity: Verbosity.Quiet, + concurrency: 1, + failFastController: controller, + }); + + expect(results[0]?.report?.verifierRejections).toBeUndefined(); prepareFiles.mockRestore(); analyzeFile.mockRestore(); @@ -1363,6 +1413,88 @@ describe('runSkillTask model lanes', () => { ); expect(result.report?.runtime).toBe('pi'); }); + + it('reports the model that actually answered when no model override is configured', async () => { + const fakeHunk = { + hunk: { newStart: 1, newCount: 10 }, + } as unknown as HunkWithContext; + const finding = makeFinding(); + + vi.spyOn(sdkRunner, 'prepareFiles').mockReturnValue({ + files: [{ filename: 'a.ts', hunks: [fakeHunk] }], + skippedFiles: [], + }); + vi.spyOn(sdkRunner, 'analyzeFile').mockResolvedValue({ + filename: 'a.ts', + findings: [finding], + usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.001 }, + failedHunks: 0, + failedExtractions: 0, + hunkFailures: [], + responseModels: ['claude-sonnet-4-5-20260929'], + } satisfies FileAnalysisResult); + vi.spyOn(sdkRunner, 'postProcessFindings').mockResolvedValue({ + findings: [finding], + auxiliaryUsage: [], + }); + + const result = await runSkillTask({ + name: 'lane-skill', + resolveSkill: async () => + ({ name: 'lane-skill', definition: '', files: [] } as unknown as SkillDefinition), + context: { + eventType: 'pull_request', + repository: { owner: 'o', name: 'n', fullName: 'o/n', defaultBranch: 'main' }, + repoPath: '/tmp', + pullRequest: { number: 1, title: 't', body: '', headSha: 'abc', baseSha: 'def', files: [] }, + } as unknown as SkillTaskOptions['context'], + runnerOptions: { + runtime: 'pi', + }, + }, 1, noopCallbacks()); + + expect(result.report?.model).toBe('claude-sonnet-4-5-20260929'); + }); + + it('reports the observed model on the error path when post-processing throws after hunks succeed', async () => { + const fakeHunk = { + hunk: { newStart: 1, newCount: 10 }, + } as unknown as HunkWithContext; + const finding = makeFinding(); + + vi.spyOn(sdkRunner, 'prepareFiles').mockReturnValue({ + files: [{ filename: 'a.ts', hunks: [fakeHunk] }], + skippedFiles: [], + }); + vi.spyOn(sdkRunner, 'analyzeFile').mockResolvedValue({ + filename: 'a.ts', + findings: [finding], + usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.001 }, + failedHunks: 0, + failedExtractions: 0, + hunkFailures: [], + responseModels: ['claude-sonnet-4-5-20260929'], + } satisfies FileAnalysisResult); + vi.spyOn(sdkRunner, 'postProcessFindings').mockRejectedValue(new Error('verification blew up')); + + const result = await runSkillTask({ + name: 'lane-skill', + resolveSkill: async () => + ({ name: 'lane-skill', definition: '', files: [] } as unknown as SkillDefinition), + context: { + eventType: 'pull_request', + repository: { owner: 'o', name: 'n', fullName: 'o/n', defaultBranch: 'main' }, + repoPath: '/tmp', + pullRequest: { number: 1, title: 't', body: '', headSha: 'abc', baseSha: 'def', files: [] }, + } as unknown as SkillTaskOptions['context'], + runnerOptions: { + runtime: 'pi', + }, + }, 1, noopCallbacks()); + + expect(result.report?.error).toBeDefined(); + expect(result.report?.model).toBe('claude-sonnet-4-5-20260929'); + }); }); describe('runSkillTask error capture', () => { diff --git a/packages/warden/src/cli/output/tasks.ts b/packages/warden/src/cli/output/tasks.ts index acffe98f7..21e04918a 100644 --- a/packages/warden/src/cli/output/tasks.ts +++ b/packages/warden/src/cli/output/tasks.ts @@ -14,6 +14,8 @@ import { analyzeFile, aggregateUsage, aggregateAuxiliaryUsage, + resolveResponseModel, + uniqueResponseModels, postProcessFindings, generateSummary, type AuxiliaryUsageEntry, @@ -47,6 +49,7 @@ interface FileProcessResult { hunkFailures: HunkFailure[]; auxiliaryUsage?: AuxiliaryUsageEntry[]; traces?: HunkTrace[]; + responseModels?: string[]; } function allAnalysisFailuresHaveCode( @@ -286,6 +289,8 @@ export async function runSkillTask( // report.skill when resolveSkill succeeded but a later step threw. // Stays undefined only if resolveSkill itself failed. let resolvedSkillName: string | undefined; + let resolvedModel: string | undefined; + let resolvedModels: string[] | undefined; try { let skill: SkillDefinition; @@ -481,6 +486,7 @@ export async function runSkillTask( hunkFailures: result.hunkFailures, auxiliaryUsage: result.auxiliaryUsage, traces: result.traces, + responseModels: result.responseModels, }; }; @@ -531,6 +537,9 @@ export async function runSkillTask( const allUsage = allResults.map((r) => r.usage).filter((u): u is UsageStats => u !== undefined); const allAuxEntries = allResults.flatMap((r) => r.auxiliaryUsage ?? []); const allTraces = allResults.flatMap((r) => r.traces ?? []); + const allResponseModels = allResults.flatMap((r) => r.responseModels ?? []); + resolvedModel = resolveResponseModel(allResponseModels, runnerOptions?.model); + resolvedModels = uniqueResponseModels(allResponseModels); const totalFailedHunks = allResults.reduce((sum, r) => sum + r.failedHunks, 0); const totalFailedExtractions = allResults.reduce((sum, r) => sum + r.failedExtractions, 0); const allHunkFailures: HunkFailure[] = allResults.flatMap((r) => r.hunkFailures); @@ -569,7 +578,8 @@ export async function runSkillTask( findings: [], usage: aggregateUsage(allUsage), durationMs: duration, - model: runnerOptions?.model, + model: resolvedModel, + models: resolvedModels, runtime, // Preserve per-file metadata (timing, partial usage, attempted // filenames) on failure runs too — `warden runs` and JSONL @@ -645,7 +655,8 @@ export async function runSkillTask( findings: finalFindings, usage: aggregateUsage(allUsage), durationMs: duration, - model: runnerOptions?.model, + model: resolvedModel, + models: resolvedModels, runtime, files: buildFileReports( preparedFiles.map((file, i) => { @@ -678,6 +689,9 @@ export async function runSkillTask( if (auxUsage) { report.auxiliaryUsage = auxUsage; } + if (processed.verifierRejections) { + report.verifierRejections = processed.verifierRejections; + } span.setAttribute('warden.finding.count', report.findings.length); // Emit metrics and log completion @@ -710,7 +724,8 @@ export async function runSkillTask( summary: `${skillName}: failed (${code})`, findings: [], durationMs: Date.now() - startTime, - model: runnerOptions?.model, + model: resolvedModel ?? runnerOptions?.model, + models: resolvedModels, runtime, error: { code, message, timestamp: new Date().toISOString() }, }; diff --git a/packages/warden/src/cli/terminal.test.ts b/packages/warden/src/cli/terminal.test.ts index 5399d3678..5285649e7 100644 --- a/packages/warden/src/cli/terminal.test.ts +++ b/packages/warden/src/cli/terminal.test.ts @@ -347,6 +347,18 @@ describe('renderTerminalReport', () => { expect(output).toContain('WARN: 1 finding extraction failed'); }); + it('renders per-skill verifierRejections warning', () => { + const report = createReport({ + durationMs: 5000, + findings: [createFinding({ severity: 'high', title: 'Bug found' })], + verifierRejections: { count: 2, reasons: ['not reproducible', 'guarded upstream'] }, + }); + + const output = renderTerminalReport([report], ciMode); + + expect(output).toContain('WARN: 2 findings rejected by verification'); + }); + it('does not render warnings when counts are zero', () => { const report = createReport({ durationMs: 5000, diff --git a/packages/warden/src/cli/terminal.ts b/packages/warden/src/cli/terminal.ts index f8b17c3d2..e802bd384 100644 --- a/packages/warden/src/cli/terminal.ts +++ b/packages/warden/src/cli/terminal.ts @@ -238,6 +238,9 @@ function renderSkillCI(report: SkillReport, verbosity: Verbosity = Verbosity.Nor if (report.failedExtractions) { lines.push(` WARN: ${report.failedExtractions} finding ${pluralize(report.failedExtractions, 'extraction')} failed`); } + if (report.verifierRejections) { + lines.push(` WARN: ${report.verifierRejections.count} ${pluralize(report.verifierRejections.count, 'finding')} rejected by verification`); + } if ((report.failedHunks || report.failedExtractions) && verbosity < Verbosity.Verbose) { lines.push(' Use -v for failure details'); } diff --git a/packages/warden/src/config/loader.test.ts b/packages/warden/src/config/loader.test.ts index ef2d1466a..a63f3e600 100644 --- a/packages/warden/src/config/loader.test.ts +++ b/packages/warden/src/config/loader.test.ts @@ -38,6 +38,35 @@ describe('resolveSkillConfigs', () => { expect(resolved?.model).toBeUndefined(); }); + it('gives two identical trigger blocks on the same skill distinct ids instead of colliding', () => { + const config: WardenConfig = { + version: 1, + skills: [ + { + name: 'test-skill', + triggers: [ + { type: 'pull_request', actions: ['opened'] }, + { type: 'pull_request', actions: ['opened'] }, + ], + }, + ], + }; + + const resolved = resolveSkillConfigs(config); + + expect(resolved).toHaveLength(2); + expect(resolved[0]?.id).not.toBe(resolved[1]?.id); + expect(resolved[0]?.skillExecutionId).not.toBe(resolved[1]?.skillExecutionId); + }); + + it('leaves a non-duplicated trigger identity unchanged', () => { + const [resolvedA] = resolveSkillConfigs(baseConfig); + const [resolvedB] = resolveSkillConfigs(baseConfig); + + expect(resolvedA?.id).toBe(resolvedB?.id); + expect(resolvedA?.skillExecutionId).toBe(resolvedB?.skillExecutionId); + }); + it('applies defaults when skill has no config', () => { const config: WardenConfig = { ...baseConfig, diff --git a/packages/warden/src/config/loader.ts b/packages/warden/src/config/loader.ts index a6346e3f2..ad74a3777 100644 --- a/packages/warden/src/config/loader.ts +++ b/packages/warden/src/config/loader.ts @@ -1,4 +1,5 @@ import { readFileSync, existsSync } from 'node:fs'; +import { createHash } from 'node:crypto'; import { join, normalize } from 'node:path'; import { parse as parseToml } from 'smol-toml'; import { Sentry } from '../sentry.js'; @@ -362,6 +363,8 @@ export function loadLayeredWardenConfig( export interface ResolvedTrigger { /** Stable replay identity derived from the skill and trigger configuration */ id: string; + /** Short stable hash of `id`, safe to use as a compact cross-artifact key */ + skillExecutionId: string; /** Skill name (used for display and deduplication) */ name: string; /** Skill reference (same as name, for downstream compatibility) */ @@ -423,6 +426,10 @@ export interface ResolvedTrigger { schedule?: ScheduleConfig; } +function deriveSkillExecutionId(identity: string): string { + return createHash('sha256').update(identity).digest('hex').slice(0, 12); +} + function triggerIdentity(skill: SkillConfig, trigger: SkillTrigger | undefined): string { return JSON.stringify({ skill: skill.name, @@ -504,6 +511,13 @@ export function resolveSkillConfigs( const defaults = config.defaults; const envModel = emptyToUndefined(process.env['WARDEN_MODEL']); const result: ResolvedTrigger[] = []; + /** Two trigger blocks resolving to the same identity (e.g. a copy-pasted duplicate) would otherwise collide on `id`/`skillExecutionId` and silently swap findings between executions during replay - disambiguate every occurrence after the first. */ + const identityOccurrences = new Map(); + const dedupeIdentity = (identity: string): string => { + const occurrence = identityOccurrences.get(identity) ?? 0; + identityOccurrences.set(identity, occurrence + 1); + return occurrence === 0 ? identity : `${identity}#${occurrence}`; + }; const runtime = defaults?.runtime ?? 'pi'; const auxiliaryModel = emptyToUndefined(defaults?.auxiliary?.model); const synthesisModel = @@ -537,8 +551,10 @@ export function resolveSkillConfigs( if (!skill.triggers || skill.triggers.length === 0) { // Wildcard: no triggers means run everywhere + const identity = dedupeIdentity(triggerIdentity(skill, undefined)); result.push({ - id: triggerIdentity(skill, undefined), + id: identity, + skillExecutionId: deriveSkillExecutionId(identity), name: skill.name, skill: skill.name, type: '*', @@ -568,8 +584,10 @@ export function resolveSkillConfigs( }); } else { for (const trigger of skill.triggers) { + const identity = dedupeIdentity(triggerIdentity(skill, trigger)); result.push({ - id: triggerIdentity(skill, trigger), + id: identity, + skillExecutionId: deriveSkillExecutionId(identity), name: skill.name, skill: skill.name, type: trigger.type, diff --git a/packages/warden/src/output/dedup.test.ts b/packages/warden/src/output/dedup.test.ts index b7f4e5736..c0ddc05c1 100644 --- a/packages/warden/src/output/dedup.test.ts +++ b/packages/warden/src/output/dedup.test.ts @@ -16,8 +16,9 @@ import { updateWardenCommentBody, consolidateBatchFindings, } from './dedup.js'; -import type { Finding } from '../types/index.js'; +import type { Finding, SkillReport } from '../types/index.js'; import type { ExistingComment } from './dedup.js'; +import { renderSkillReport } from './renderer.js'; describe('generateContentHash', () => { it('generates consistent 8-char hex hash', () => { @@ -357,7 +358,8 @@ describe('deduplicateFindings', () => { expect(result.duplicateActions).toHaveLength(1); expect(result.duplicateActions[0]!.type).toBe('update_warden'); expect(result.duplicateActions[0]!.matchType).toBe('hash'); - expect(result.duplicateActions[0]!.finding.id).toBe('WRZ-XPL'); + expect(result.duplicateActions[0]!.finding.id).toBe(baseFinding.id); + expect(result.duplicateActions[0]!.finding.reportedId).toBe('WRZ-XPL'); }); it('keeps findings with different content', async () => { @@ -626,6 +628,23 @@ describe('findingToExistingComment', () => { expect(comment!.skills).toEqual(['security-review']); }); + it('includes skillExecutionId when provided', () => { + const finding: Finding = { + id: 'f1', + severity: 'high', + title: 'SQL Injection', + description: 'User input passed to query', + location: { + path: 'src/db.ts', + startLine: 42, + }, + }; + + const comment = findingToExistingComment(finding, 'security-review', 'exec-1'); + expect(comment).not.toBeNull(); + expect(comment!.skillExecutionId).toBe('exec-1'); + }); + it('uses startLine when endLine is not set', () => { const finding: Finding = { id: 'f1', @@ -674,6 +693,27 @@ ${marker}`; const parsed = parseMarker(body); expect(parsed).toEqual({ path, line, contentHash: hash }); }); + + it('parses skill and finding id back out of the real rendered attribution footer', () => { + const report: SkillReport = { + skill: 'security-review', + summary: 'Found 1 issue', + findings: [ + { + id: 'sql-injection-1', + severity: 'high', + title: 'SQL Injection', + description: 'User input passed directly to query', + location: { path: 'src/db.ts', startLine: 42, endLine: 45 }, + }, + ], + }; + + const body = renderSkillReport(report).review!.comments[0]!.body; + + expect(parseWardenFindingId(body)).toBe('sql-injection-1'); + expect(parseWardenSkills(body)).toEqual(['security-review']); + }); }); describe('finding metadata', () => { diff --git a/packages/warden/src/output/dedup.ts b/packages/warden/src/output/dedup.ts index 3a0a553e1..f0c40b96e 100644 --- a/packages/warden/src/output/dedup.ts +++ b/packages/warden/src/output/dedup.ts @@ -42,6 +42,8 @@ export interface ExistingComment { isWarden?: boolean; /** Skills that have already detected this issue (for Warden comments) */ skills?: string[]; + /** The skillExecutionId that produced this comment, when posted earlier in the current run (unavailable for comments fetched from GitHub) */ + skillExecutionId?: string; /** Original finding severity, when emitted by a recent Warden comment */ severity?: Severity; /** Original finding confidence, when emitted by a recent Warden comment */ @@ -66,8 +68,6 @@ export type DuplicateActionType = 'update_warden' | 'react_external'; */ export interface DuplicateAction { type: DuplicateActionType; - /** ID produced by the current run before any Warden ID recentering */ - originalFindingId: string; finding: Finding; existingComment: ExistingComment; /** Whether this was a hash match or semantic match */ @@ -659,7 +659,11 @@ export async function processDuplicateActions( * Convert a Finding to an ExistingComment for cross-trigger deduplication. * Returns null if the finding has no location. */ -export function findingToExistingComment(finding: Finding, skill?: string): ExistingComment | null { +export function findingToExistingComment( + finding: Finding, + skill?: string, + skillExecutionId?: string +): ExistingComment | null { if (!finding.location) { return null; } @@ -670,9 +674,10 @@ export function findingToExistingComment(finding: Finding, skill?: string): Exis line: finding.location.endLine ?? finding.location.startLine, title: finding.title, description: finding.description, - findingId: finding.id, + findingId: finding.reportedId ?? finding.id, contentHash: generateContentHash(finding.title, finding.description), isWarden: true, + skillExecutionId, skills: skill ? [skill] : [], severity: finding.severity, ...(finding.confidence ? { confidence: finding.confidence } : {}), @@ -915,11 +920,10 @@ export async function deduplicateFindings( if (matchingComment) { const duplicateFinding = matchingComment.isWarden && matchingComment.findingId - ? { ...finding, id: matchingComment.findingId } + ? { ...finding, reportedId: matchingComment.findingId } : finding; duplicateActions.push({ type: matchingComment.isWarden ? 'update_warden' : 'react_external', - originalFindingId: finding.id, finding: duplicateFinding, existingComment: matchingComment, matchType: 'hash', @@ -950,11 +954,10 @@ export async function deduplicateFindings( const matchingComment = semanticResult.matches.get(finding.id); if (matchingComment) { const duplicateFinding = matchingComment.isWarden && matchingComment.findingId - ? { ...finding, id: matchingComment.findingId } + ? { ...finding, reportedId: matchingComment.findingId } : finding; duplicateActions.push({ type: matchingComment.isWarden ? 'update_warden' : 'react_external', - originalFindingId: finding.id, finding: duplicateFinding, existingComment: matchingComment, matchType: 'semantic', diff --git a/packages/warden/src/output/github-checks.ts b/packages/warden/src/output/github-checks.ts index b97ab0bfc..0d0a2e7ea 100644 --- a/packages/warden/src/output/github-checks.ts +++ b/packages/warden/src/output/github-checks.ts @@ -1,7 +1,7 @@ import type { Octokit } from '@octokit/rest'; import { SEVERITY_ORDER, filterFindings } from '../types/index.js'; import type { Severity, SeverityThreshold, ConfidenceThreshold, Finding, SkillReport, UsageStats, AuxiliaryUsageMap } from '../types/index.js'; -import { formatDuration, formatCost, formatTokens, totalUsageCost, totalUsageStats } from '../cli/output/formatters.js'; +import { displayFindingId, formatDuration, formatCost, formatTokens, totalUsageCost, totalUsageStats } from '../cli/output/formatters.js'; import { escapeHtml } from '../utils/index.js'; /** @@ -160,7 +160,7 @@ export function findingsToAnnotations(findings: Finding[], reportOn?: SeverityTh end_line: loc.endLine ?? loc.startLine, annotation_level: severityToAnnotationLevel(finding.severity), message: escapeHtml(finding.description), - title: `[${finding.id}] ${escapeHtml(finding.title)} (additional location)`, + title: `[${displayFindingId(finding)}] ${escapeHtml(finding.title)} (additional location)`, }); } } diff --git a/packages/warden/src/output/issue-renderer.ts b/packages/warden/src/output/issue-renderer.ts index f1405bdc3..153622877 100644 --- a/packages/warden/src/output/issue-renderer.ts +++ b/packages/warden/src/output/issue-renderer.ts @@ -1,6 +1,6 @@ import { SEVERITY_ORDER } from '../types/index.js'; import type { SkillReport, Finding, Severity } from '../types/index.js'; -import { capitalize, countBySeverity } from '../cli/output/formatters.js'; +import { capitalize, countBySeverity, displayFindingId } from '../cli/output/formatters.js'; import { escapeHtml } from '../utils/index.js'; export interface IssueRenderOptions { @@ -156,7 +156,7 @@ function renderFindingItem(finding: Finding, ctx: LinkContext): string { } } - let line = `- \`${finding.id}\` **${escapeHtml(finding.title)}**${locationStr} · ${finding.severity}`; + let line = `- \`${displayFindingId(finding)}\` **${escapeHtml(finding.title)}**${locationStr} · ${finding.severity}`; line += `\n ${escapeHtml(finding.description)}`; return line; diff --git a/packages/warden/src/output/renderer.test.ts b/packages/warden/src/output/renderer.test.ts index f4089a57f..fa9ec1d93 100644 --- a/packages/warden/src/output/renderer.test.ts +++ b/packages/warden/src/output/renderer.test.ts @@ -46,6 +46,7 @@ describe('renderSkillReport', () => { expect(review.comments[0]!.path).toBe('src/db.ts'); expect(review.comments[0]!.line).toBe(45); expect(review.comments[0]!.body).toContain('SQL Injection'); + expect(review.comments[0]!.findingId).toBe('sql-injection-1'); }); it('includes skill attribution footnote in comments', () => { @@ -76,6 +77,33 @@ describe('renderSkillReport', () => { expect(result.review!.comments[0]!.body).not.toContain('`f1`'); }); + it('shows reportedId instead of id in the attribution footnote once dedupe sets it', () => { + const report: SkillReport = { + ...baseReport, + skill: 'code-review', + findings: [ + { + id: 'f1', + reportedId: 'WRZ-XPL', + severity: 'medium', + title: 'Issue', + description: 'Details', + location: { + path: 'src/file.ts', + startLine: 10, + }, + }, + ], + }; + + const result = renderSkillReport(report); + + expect(result.review!.comments[0]!.body).toContain( + 'Identified by Warden · code-review · WRZ-XPL' + ); + expect(result.review!.comments[0]!.body).not.toContain('· f1<'); + }); + it('does not include confidence in attribution footnote', () => { const report: SkillReport = { ...baseReport, diff --git a/packages/warden/src/output/renderer.ts b/packages/warden/src/output/renderer.ts index e8c6ef198..331e28962 100644 --- a/packages/warden/src/output/renderer.ts +++ b/packages/warden/src/output/renderer.ts @@ -1,7 +1,7 @@ import { SEVERITY_ORDER, filterFindings } from '../types/index.js'; import type { SkillReport, Finding, Severity, SeverityThreshold } from '../types/index.js'; import type { RenderResult, RenderOptions, GitHubReview, GitHubComment } from './types.js'; -import { capitalize, formatStatsCompact, countBySeverity, pluralize } from '../cli/output/formatters.js'; +import { capitalize, displayFindingId, formatStatsCompact, countBySeverity, pluralize } from '../cli/output/formatters.js'; import { generateContentHash, generateFindingMetadata, generateMarker } from './dedup.js'; import { escapeHtml } from '../utils/index.js'; @@ -88,7 +88,7 @@ function renderReview( } // Add attribution footer with skill name and finding ID - body += `\n\n${renderAttributionFooter(report.skill, finding.id)}`; + body += `\n\n${renderAttributionFooter(report.skill, displayFindingId(finding))}`; // Add deduplication marker const contentHash = generateContentHash(finding.title, finding.description); @@ -105,6 +105,7 @@ function renderReview( side: 'RIGHT' as const, start_line: isMultiLine ? location.startLine : undefined, start_side: isMultiLine ? ('RIGHT' as const) : undefined, + findingId: finding.id, }; }); @@ -249,7 +250,7 @@ function renderFindingItem(finding: Finding): string { const extra = finding.additionalLocations?.length ? ` (+${finding.additionalLocations.length} more ${pluralize(finding.additionalLocations.length, 'location')})` : ''; - return `- \`${finding.id}\` **${escapeHtml(finding.title)}**${location}${extra} · ${finding.severity}: ${escapeHtml(finding.description)}`; + return `- \`${displayFindingId(finding)}\` **${escapeHtml(finding.title)}**${location}${extra} · ${finding.severity}: ${escapeHtml(finding.description)}`; } /** Render findings as markdown for inclusion in a review body. */ diff --git a/packages/warden/src/output/types.ts b/packages/warden/src/output/types.ts index 3893c31ba..ada037e6d 100644 --- a/packages/warden/src/output/types.ts +++ b/packages/warden/src/output/types.ts @@ -10,6 +10,8 @@ export interface GitHubComment { side?: 'LEFT' | 'RIGHT'; start_line?: number; start_side?: 'LEFT' | 'RIGHT'; + /** The finding this comment renders, for matching back to the real comment id/url once posted. */ + findingId?: string; } export interface GitHubReview { diff --git a/packages/warden/src/sdk/analyze.test.ts b/packages/warden/src/sdk/analyze.test.ts index 4ff3b1ecb..ed82b9362 100644 --- a/packages/warden/src/sdk/analyze.test.ts +++ b/packages/warden/src/sdk/analyze.test.ts @@ -375,6 +375,54 @@ describe('analyzeFile', () => { consoleSpy.mockRestore(); }); + it('preserves the response model when the circuit opens on an SDK-reported provider error', async () => { + const controller = new AbortController(); + const circuitBreaker = new ProviderFailureCircuitBreaker({ + maxConsecutiveProviderFailures: 1, + abortController: controller, + }); + const runSkill = vi.fn().mockResolvedValue({ + result: { + status: 'provider_error', + text: '', + errors: ['upstream provider error'], + usage: makeUsage(), + responseModel: 'claude-sonnet-4-5-20260929', + }, + }); + vi.mocked(getRuntime).mockReturnValue({ + name: 'claude', + runSkill, + runAuxiliary: vi.fn(), + runSynthesis: vi.fn(), + } as unknown as Runtime); + const skill: SkillDefinition = { + name: 'security-review', + description: 'Security review.', + prompt: 'Return findings as JSON.', + }; + + const result = await analyzeFile( + skill, + makePreparedFile(1), + '/tmp/repo', + { + abortController: controller, + circuitBreaker, + retry: { + maxRetries: 0, + initialDelayMs: 1, + backoffMultiplier: 1, + maxDelayMs: 1, + }, + }, + ); + + expect(circuitBreaker.reason?.code).toBe('provider_unavailable'); + expect(result.failedHunks).toBe(1); + expect(result.responseModels).toEqual(['claude-sonnet-4-5-20260929']); + }); + it('opens the shared circuit after consecutive provider failures', async () => { const controller = new AbortController(); const circuitBreaker = new ProviderFailureCircuitBreaker({ @@ -723,6 +771,118 @@ describe('runSkill', () => { expect(report.runtime).toBe('pi'); }); + it('reports the model that actually answered when no model override is configured', async () => { + const runSkillMock = vi.fn().mockResolvedValue({ + result: { + status: 'success', + text: JSON.stringify({ findings: [] }), + errors: [], + usage: makeUsage(), + responseModel: 'claude-sonnet-4-5-20260929', + }, + }); + vi.mocked(getRuntime).mockReturnValue({ + name: 'claude', + runSkill: runSkillMock, + runAuxiliary: vi.fn(), + runSynthesis: vi.fn(), + } as unknown as Runtime); + + const report = await runSkill( + { + name: 'security-review', + description: 'Security review.', + prompt: 'Return findings as JSON.', + }, + makeContextWithOneHunk(), + {}, + ); + + expect(report.model).toBe('claude-sonnet-4-5-20260929'); + expect(report.models).toBeUndefined(); + }); + + it('falls back to the configured model and reports the full set when hunks disagree', async () => { + const runSkillMock = vi.fn() + .mockResolvedValueOnce({ + result: { + status: 'success', + text: JSON.stringify({ findings: [] }), + errors: [], + usage: makeUsage(), + responseModel: 'claude-sonnet-4-5-20260929', + }, + }) + .mockResolvedValueOnce({ + result: { + status: 'success', + text: JSON.stringify({ findings: [] }), + errors: [], + usage: makeUsage(), + responseModel: 'claude-opus-4-8-20260601', + }, + }); + vi.mocked(getRuntime).mockReturnValue({ + name: 'claude', + runSkill: runSkillMock, + runAuxiliary: vi.fn(), + runSynthesis: vi.fn(), + } as unknown as Runtime); + + const report = await runSkill( + { + name: 'security-review', + description: 'Security review.', + prompt: 'Return findings as JSON.', + }, + makeContextWithTwoHunks(), + { model: 'configured-alias' }, + ); + + expect(report.model).toBe('configured-alias'); + expect(report.models).toEqual(['claude-opus-4-8-20260601', 'claude-sonnet-4-5-20260929']); + }); + + it('attributes each skill its own response model when multiple skills run concurrently', async () => { + const modelBySkill: Record = { + 'skill-a': 'model-a', + 'skill-b': 'model-b', + 'skill-c': 'model-c', + }; + const runSkillMock = vi.fn(async ({ skillName }: { skillName: string }) => ({ + result: { + status: 'success', + text: JSON.stringify({ findings: [] }), + errors: [], + usage: makeUsage(), + responseModel: modelBySkill[skillName], + }, + })); + vi.mocked(getRuntime).mockReturnValue({ + name: 'claude', + runSkill: runSkillMock, + runAuxiliary: vi.fn(), + runSynthesis: vi.fn(), + } as unknown as Runtime); + + const [reportA, reportB, reportC] = await Promise.all( + Object.keys(modelBySkill).map((skillName) => + runSkill( + { name: skillName, description: `${skillName} description.`, prompt: 'Return findings as JSON.' }, + makeContextWithOneHunk(), + {}, + ) + ) + ); + + expect(reportA!.skill).toBe('skill-a'); + expect(reportA!.model).toBe('model-a'); + expect(reportB!.skill).toBe('skill-b'); + expect(reportB!.model).toBe('model-b'); + expect(reportC!.skill).toBe('skill-c'); + expect(reportC!.model).toBe('model-c'); + }); + it('preserves candidate findings when verification is interrupted', async () => { const controller = new AbortController(); const runSkillMock = vi.fn() diff --git a/packages/warden/src/sdk/analyze.ts b/packages/warden/src/sdk/analyze.ts index b3048d73c..bd0b24140 100644 --- a/packages/warden/src/sdk/analyze.ts +++ b/packages/warden/src/sdk/analyze.ts @@ -7,7 +7,7 @@ import { SkillRunnerError, WardenAuthenticationError, isRetryableError, isAuthen import { genAiProviderName } from './otel.js'; import type { CircuitBreakerReason } from './circuit-breaker.js'; import { DEFAULT_RETRY_CONFIG, calculateRetryDelay, sleep } from './retry.js'; -import { aggregateUsage, emptyUsage, estimateTokens, aggregateAuxiliaryUsage, aggregateAuxiliaryUsageAttribution } from './usage.js'; +import { aggregateUsage, emptyUsage, estimateTokens, aggregateAuxiliaryUsage, aggregateAuxiliaryUsageAttribution, resolveResponseModel, uniqueResponseModels } from './usage.js'; import { buildHunkSystemPrompt, buildHunkUserPrompt, type PRPromptContext } from './prompt.js'; import { extractFindingsJson, extractFindingsWithLLM, validateFindings } from './extract.js'; import { postProcessFindings } from './post-process.js'; @@ -27,7 +27,7 @@ import { type ChunkAnalysisResult, } from './types.js'; import { prepareFiles } from './prepare.js'; -import type { EventContext, SkillReport, UsageStats, HunkFailure, HunkTrace } from '../types/index.js'; +import type { EventContext, SkillReport, UsageStats, HunkFailure, HunkTrace, VerifierRejections } from '../types/index.js'; import type { SourceSnippet, SourceSnippetLine } from '../types/index.js'; import { runPool } from '../utils/index.js'; import { getSpanContext, startTraceRecorder, withTraceRecorder, type TraceRecorder } from '../sentry-trace.js'; @@ -72,6 +72,7 @@ function hunkFailureFromCircuit( usage: UsageStats[], attempts: number, trace?: HunkTrace, + responseModel?: string, ): HunkAnalysisResult { return { findings: [], @@ -82,6 +83,7 @@ function hunkFailureFromCircuit( failureMessage: reason.message, attempts, trace, + responseModel, }; } @@ -504,6 +506,7 @@ async function analyzeHunk( result: resultMessage, traceRecorder, }), + resultMessage.responseModel, ); } return { @@ -514,6 +517,7 @@ async function analyzeHunk( failureCode, failureMessage, attempts: attempt + 1, + responseModel: resultMessage.responseModel, trace: buildHunkTrace({ enabled: options.captureTraces, span, @@ -588,6 +592,7 @@ async function analyzeHunk( runtime: runtimeName, }] : undefined, + responseModel: resultMessage.responseModel, trace: buildHunkTrace({ enabled: options.captureTraces, span, @@ -829,6 +834,7 @@ export async function analyzeFile( const fileAuxiliaryUsage: AuxiliaryUsageEntry[] = []; const hunkFailures: HunkFailure[] = []; const hunkTraces: HunkTrace[] = []; + const fileResponseModels: string[] = []; let failedHunks = 0; let failedExtractions = 0; @@ -886,6 +892,9 @@ export async function analyzeFile( if (result.trace) { hunkTraces.push(result.trace); } + if (result.responseModel) { + fileResponseModels.push(result.responseModel); + } const chunkResult: ChunkAnalysisResult = { filename: file.filename, model: options.model, @@ -926,6 +935,7 @@ export async function analyzeFile( hunkFailures, auxiliaryUsage: fileAuxiliaryUsage.length > 0 ? fileAuxiliaryUsage : undefined, traces: hunkTraces.length > 0 ? hunkTraces : undefined, + responseModels: fileResponseModels.length > 0 ? fileResponseModels : undefined, }; }, ); @@ -1031,6 +1041,7 @@ async function runSkillAnalysis( const allUsage: UsageStats[] = []; const allAuxiliaryUsage: AuxiliaryUsageEntry[] = []; const allTraces: HunkTrace[] = []; + const allResponseModels: string[] = []; // Track failed hunks across all files let totalFailedHunks = 0; @@ -1160,6 +1171,9 @@ async function runSkillAnalysis( if (fr.result.traces) { allTraces.push(...fr.result.traces); } + if (fr.result.responseModels) { + allResponseModels.push(...fr.result.responseModels); + } } // All hunks failed — typically a systemic problem (auth, subprocess, etc). @@ -1205,6 +1219,7 @@ async function runSkillAnalysis( } let finalFindings = allFindings; + let verifierRejections: VerifierRejections | undefined; if (options.postProcessFindings !== false) { const processed = await postProcessFindings(allFindings, { skill, @@ -1224,6 +1239,7 @@ async function runSkillAnalysis( }); finalFindings = processed.findings; allAuxiliaryUsage.push(...processed.auxiliaryUsage); + verifierRejections = processed.verifierRejections; } // Generate summary @@ -1238,7 +1254,8 @@ async function runSkillAnalysis( findings: finalFindings, usage: totalUsage, durationMs: Date.now() - startTime, - model: options.model, + model: resolveResponseModel(allResponseModels, options.model), + models: uniqueResponseModels(allResponseModels), files: buildFileReports( fileResults.map((fr) => ({ filename: fr.filename, @@ -1272,5 +1289,8 @@ async function runSkillAnalysis( if (auxAttribution) { report.auxiliaryUsageAttribution = auxAttribution; } + if (verifierRejections) { + report.verifierRejections = verifierRejections; + } return report; } diff --git a/packages/warden/src/sdk/extract.ts b/packages/warden/src/sdk/extract.ts index b7ef0229a..47ff27822 100644 --- a/packages/warden/src/sdk/extract.ts +++ b/packages/warden/src/sdk/extract.ts @@ -575,6 +575,8 @@ Singletons should not appear. Return [] if no findings describe the same issue.` finding, replacement: findReplacementForAbsorbed(finding, replacements), reason: 'same root cause at another location', + model: options?.model, + runtime: options?.runtime, }); } diff --git a/packages/warden/src/sdk/post-process.ts b/packages/warden/src/sdk/post-process.ts index 1660ab93d..d41d2419f 100644 --- a/packages/warden/src/sdk/post-process.ts +++ b/packages/warden/src/sdk/post-process.ts @@ -1,6 +1,6 @@ import type { Effort, SkillDefinition } from '../config/schema.js'; import { emitDedupMetrics } from '../sentry.js'; -import type { Finding } from '../types/index.js'; +import type { Finding, VerifierRejections } from '../types/index.js'; import { deduplicateFindings, mergeCrossLocationFindings } from './extract.js'; import type { PromptPRContext } from './prompt-sections.js'; import type { RuntimeName } from './runtimes/index.js'; @@ -27,6 +27,7 @@ export interface PostProcessFindingsOptions { export interface PostProcessFindingsResult { findings: Finding[]; auxiliaryUsage: AuxiliaryUsageEntry[]; + verifierRejections?: VerifierRejections; } /** @@ -42,6 +43,7 @@ export async function postProcessFindings( emitDedupMetrics(options.skill.name, findings.length, uniqueFindings.length); let currentFindings = uniqueFindings; + let verifierRejections: VerifierRejections | undefined; if (options.verifyFindings !== false) { const verification = await verifyFindings(currentFindings, { repoPath: options.repoPath, @@ -57,6 +59,7 @@ export async function postProcessFindings( onFindingProcessing: options.onFindingProcessing, }); currentFindings = verification.findings; + verifierRejections = verification.verifierRejections; if (verification.usage) { auxiliaryUsage.push({ agent: 'verification', @@ -86,5 +89,5 @@ export async function postProcessFindings( }); } - return { findings: currentFindings, auxiliaryUsage }; + return { findings: currentFindings, auxiliaryUsage, verifierRejections }; } diff --git a/packages/warden/src/sdk/runner.ts b/packages/warden/src/sdk/runner.ts index 22b0dac0e..782bc5d0d 100644 --- a/packages/warden/src/sdk/runner.ts +++ b/packages/warden/src/sdk/runner.ts @@ -22,7 +22,7 @@ export { verifyAuth } from './auth.js'; export { calculateRetryDelay } from './retry.js'; // Re-export usage utilities -export { aggregateUsage, aggregateAuxiliaryUsage, mergeAuxiliaryUsage, estimateTokens } from './usage.js'; +export { aggregateUsage, aggregateAuxiliaryUsage, mergeAuxiliaryUsage, estimateTokens, resolveResponseModel, uniqueResponseModels } from './usage.js'; // Re-export pricing utilities export { anthropicUsageToStats, apiUsageToStats } from './pricing.js'; diff --git a/packages/warden/src/sdk/types.ts b/packages/warden/src/sdk/types.ts index 3e59248ba..8914ea2b5 100644 --- a/packages/warden/src/sdk/types.ts +++ b/packages/warden/src/sdk/types.ts @@ -14,10 +14,13 @@ export interface AuxiliaryUsageEntry { export interface FindingProcessingEvent { stage: 'dedupe' | 'verification' | 'merge' | 'fix_gate'; - action: 'dropped' | 'rejected' | 'revised' | 'merged' | 'stripped_fix'; + action: 'dropped' | 'rejected' | 'revised' | 'kept' | 'merged' | 'stripped_fix'; finding: Finding; reason?: string; replacement?: Finding; + /** Model that produced this stage's verdict/replacement, when known. */ + model?: string; + runtime?: RuntimeName; } /** Default concurrency for file-level parallel processing (standalone SDK usage only) */ @@ -48,6 +51,8 @@ export interface HunkAnalysisResult { auxiliaryUsage?: AuxiliaryUsageEntry[]; /** Optional runtime trace captured for this hunk. */ trace?: HunkTrace; + /** Model that actually answered this hunk, from the live API response. */ + responseModel?: string; } /** Result from one completed chunk, suitable for durable run logging. */ @@ -223,6 +228,8 @@ export interface FileAnalysisResult { auxiliaryUsage?: AuxiliaryUsageEntry[]; /** Optional runtime traces captured for analyzed hunks. */ traces?: HunkTrace[]; + /** Models that actually answered this file's hunks, from the live API responses. */ + responseModels?: string[]; } /** diff --git a/packages/warden/src/sdk/usage.ts b/packages/warden/src/sdk/usage.ts index b26161228..a02aa31b8 100644 --- a/packages/warden/src/sdk/usage.ts +++ b/packages/warden/src/sdk/usage.ts @@ -113,6 +113,18 @@ function uniqueSorted(values: (string | undefined)[]): string[] | undefined { return unique.length > 0 ? unique : undefined; } +/** Collapses a run's observed response models to a single value when they agree, or falls back. */ +export function resolveResponseModel(models: string[], fallback?: string): string | undefined { + const unique = [...new Set(models)]; + return unique.length === 1 ? unique[0] : fallback; +} + +/** Distinct response models observed across hunks, only when they disagree (>1 distinct value). */ +export function uniqueResponseModels(models: string[]): string[] | undefined { + const unique = uniqueSorted(models); + return unique && unique.length > 1 ? unique : undefined; +} + function attributionFromEntries(entries: AuxiliaryUsageEntry[]): UsageAttribution | undefined { const models = uniqueSorted(entries.map((entry) => entry.model)); const runtimes = uniqueSorted(entries.map((entry) => entry.runtime)); diff --git a/packages/warden/src/sdk/verify.test.ts b/packages/warden/src/sdk/verify.test.ts index 52859b8bf..7243e5ec3 100644 --- a/packages/warden/src/sdk/verify.test.ts +++ b/packages/warden/src/sdk/verify.test.ts @@ -104,8 +104,11 @@ describe('verifyFindings', () => { action: 'rejected', finding, reason: 'guarded upstream', + model: 'claude-haiku-4-5', + runtime: undefined, }); expect(result.usage).toEqual(expect.objectContaining(makeUsage())); + expect(result.verifierRejections).toEqual({ count: 1, reasons: ['guarded upstream'] }); expect(runtime.runSkill).toHaveBeenCalledWith(expect.objectContaining({ repoPath: '/repo', skillName: 'test-skill:verification', @@ -129,6 +132,52 @@ describe('verifyFindings', () => { })); }); + it('truncates a long rejection reason to 500 characters', async () => { + const longReason = 'x'.repeat(600); + const runtime = mockRuntime(JSON.stringify({ verdict: 'reject', reason: longReason })); + vi.mocked(getRuntime).mockReturnValue(runtime); + + const finding = makeFinding(); + const result = await verifyFindings([finding], { + repoPath: '/repo', + skill: makeSkill(), + model: 'claude-haiku-4-5', + prContext: { + repository: 'getsentry/sentry', + title: 'Fix guarded path', + body: 'Adds a guard before the call.', + changedFiles: ['src/app.ts'], + }, + }); + + expect(result.verifierRejections?.reasons[0]).toHaveLength(500); + expect(result.verifierRejections?.reasons[0]).toBe(longReason.slice(0, 500)); + }); + + it('emits a kept processing event when the verifier returns keep', async () => { + const runtime = mockRuntime('{"verdict":"keep","reason":"still real after tracing"}'); + vi.mocked(getRuntime).mockReturnValue(runtime); + const onFindingProcessing = vi.fn(); + + const finding = makeFinding(); + const result = await verifyFindings([finding], { + repoPath: '/repo', + skill: makeSkill(), + model: 'claude-haiku-4-5', + onFindingProcessing, + }); + + expect(result.findings).toEqual([finding]); + expect(onFindingProcessing).toHaveBeenCalledWith({ + stage: 'verification', + action: 'kept', + finding, + reason: 'still real after tracing', + model: 'claude-haiku-4-5', + runtime: undefined, + }); + }); + it('keeps the original id when revising a finding', async () => { const revised = makeFinding({ id: 'DIFFERENT', @@ -198,6 +247,31 @@ describe('verifyFindings', () => { expect(verified?.additionalLocations).toBeUndefined(); }); + it('emits a kept processing event, not revised, when a revise verdict has no usable finding', async () => { + const runtime = mockRuntime(JSON.stringify({ + verdict: 'revise', + finding: null, + reason: 'impact is narrower', + })); + vi.mocked(getRuntime).mockReturnValue(runtime); + const onFindingProcessing = vi.fn(); + + const finding = makeFinding(); + const result = await verifyFindings([finding], { + repoPath: '/repo', + skill: makeSkill(), + onFindingProcessing, + }); + + expect(result.findings).toEqual([finding]); + expect(onFindingProcessing).toHaveBeenCalledWith(expect.objectContaining({ + stage: 'verification', + action: 'kept', + finding, + })); + expect(onFindingProcessing).not.toHaveBeenCalledWith(expect.objectContaining({ action: 'revised' })); + }); + it('accepts verifier JSON when verdict is not the first key', async () => { const runtime = mockRuntime('{"reason":"guarded upstream","verdict":"reject"}'); vi.mocked(getRuntime).mockReturnValue(runtime); @@ -364,4 +438,56 @@ describe('verifyFindings', () => { tools: { allowed: ['Read', 'Grep', 'WebFetch'] }, })); }); + + it('omits verifierRejections when nothing is rejected', async () => { + const runtime = mockRuntime('{"verdict":"keep"}'); + vi.mocked(getRuntime).mockReturnValue(runtime); + + const result = await verifyFindings([makeFinding()], { + repoPath: '/repo', + skill: makeSkill(), + }); + + expect(result.verifierRejections).toBeUndefined(); + }); + + it('falls back to a default reason when the verifier rejects without one', async () => { + const runtime = mockRuntime('{"verdict":"reject"}'); + vi.mocked(getRuntime).mockReturnValue(runtime); + + const result = await verifyFindings([makeFinding()], { + repoPath: '/repo', + skill: makeSkill(), + }); + + expect(result.verifierRejections).toEqual({ count: 1, reasons: ['No reason provided'] }); + }); + + it('aggregates rejections across multiple findings', async () => { + const runtime: Runtime = { + name: 'claude', + runSkill: vi.fn() + .mockResolvedValueOnce({ + result: { status: 'success', text: '{"verdict":"reject","reason":"first"}', errors: [], usage: makeUsage() }, + }) + .mockResolvedValueOnce({ + result: { status: 'success', text: '{"verdict":"keep"}', errors: [], usage: makeUsage() }, + }) + .mockResolvedValueOnce({ + result: { status: 'success', text: '{"verdict":"reject","reason":"second"}', errors: [], usage: makeUsage() }, + }), + runAuxiliary: vi.fn(), + runSynthesis: vi.fn(), + } as unknown as Runtime; + vi.mocked(getRuntime).mockReturnValue(runtime); + + const findings = [makeFinding({ id: 'A' }), makeFinding({ id: 'B' }), makeFinding({ id: 'C' })]; + const result = await verifyFindings(findings, { + repoPath: '/repo', + skill: makeSkill(), + }); + + expect(result.findings).toHaveLength(1); + expect(result.verifierRejections).toEqual({ count: 2, reasons: ['first', 'second'] }); + }); }); diff --git a/packages/warden/src/sdk/verify.ts b/packages/warden/src/sdk/verify.ts index 440ead650..41cd7ca7c 100644 --- a/packages/warden/src/sdk/verify.ts +++ b/packages/warden/src/sdk/verify.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; import type { Effort, SkillDefinition } from '../config/schema.js'; -import { FindingSchema, type Finding, type UsageStats } from '../types/index.js'; +import { FindingSchema, type Finding, type UsageStats, type VerifierRejections } from '../types/index.js'; import { aggregateUsage } from './usage.js'; import { extractBalancedJson } from './extract.js'; import { @@ -44,6 +44,7 @@ export interface VerifyFindingsOptions { export interface VerifyFindingsResult { findings: Finding[]; usage?: UsageStats; + verifierRejections?: VerifierRejections; } const VerificationVerdictSchema = z.object({ @@ -57,10 +58,12 @@ type VerificationVerdict = z.infer; interface VerificationTaskResult { finding?: Finding; usage?: UsageStats; + rejectionReason?: string; } const JSON_OBJECT_START = /\{/g; const VERIFICATION_CONCURRENCY = 4; +const MAX_REJECTION_REASON_LENGTH = 500; function isAbortRequested(error: unknown, abortController?: AbortController): boolean { return (abortController?.signal.aborted ?? false) || classifyError(error).code === 'aborted'; @@ -207,17 +210,36 @@ function notifyVerdict( action: 'rejected', finding, reason: verdict.reason, + model: options.model, + runtime: options.runtime, }); return; } - if (verdict.verdict === 'revise' && next) { + if (verdict.verdict === 'revise' && next && next !== finding) { options.onFindingProcessing?.({ stage: 'verification', action: 'revised', finding, replacement: next, reason: verdict.reason, + model: options.model, + runtime: options.runtime, + }); + return; + } + + // 'revise' falls through here when applyVerdict couldn't produce a usable + // replacement (missing/invalid verdict.finding) and returned the original + // finding unchanged - that's a no-op, not a real revision. + if (verdict.verdict === 'keep' || verdict.verdict === 'revise') { + options.onFindingProcessing?.({ + stage: 'verification', + action: 'kept', + finding, + reason: verdict.reason, + model: options.model, + runtime: options.runtime, }); } } @@ -277,7 +299,10 @@ export async function verifyFindings( : null; const next = applyVerdict(finding, verdict); notifyVerdict(options, finding, verdict, next); - return { finding: next ?? undefined, usage: result?.usage }; + const rejectionReason = verdict?.verdict === 'reject' + ? (verdict.reason ?? 'No reason provided').slice(0, MAX_REJECTION_REASON_LENGTH) + : undefined; + return { finding: next ?? undefined, usage: result?.usage, rejectionReason }; } catch (error) { if (isAbortRequested(error, options.abortController)) { return keepFindingAfterInterruptedVerification(finding); @@ -307,9 +332,15 @@ export async function verifyFindings( const verified = results.flatMap((result) => result.finding ? [result.finding] : []); const usage = results.map((result) => result.usage).filter((u): u is UsageStats => u !== undefined); + const rejectionReasons = results + .map((result) => result.rejectionReason) + .filter((reason): reason is string => reason !== undefined); return { findings: verified, usage: usage.length > 0 ? aggregateUsage(usage) : undefined, + verifierRejections: rejectionReasons.length > 0 + ? { count: rejectionReasons.length, reasons: rejectionReasons } + : undefined, }; } diff --git a/packages/warden/src/triggers/matcher.test.ts b/packages/warden/src/triggers/matcher.test.ts index 360588f3d..63f431d33 100644 --- a/packages/warden/src/triggers/matcher.test.ts +++ b/packages/warden/src/triggers/matcher.test.ts @@ -138,6 +138,7 @@ describe('matchTrigger', () => { const baseTrigger: ResolvedTrigger = { id: 'test-trigger-id', + skillExecutionId: 'test-trigger-id', name: 'test-trigger', skill: 'test-skill', type: 'pull_request', diff --git a/packages/warden/src/triggers/matcher.ts b/packages/warden/src/triggers/matcher.ts index b2ee46bbc..5955acea1 100644 --- a/packages/warden/src/triggers/matcher.ts +++ b/packages/warden/src/triggers/matcher.ts @@ -93,7 +93,7 @@ export function matchGlob(pattern: string, path: string): boolean { * Check if a file list matches the path filters. * Returns true if paths match (or no filters), false if all files are excluded. */ -function matchPathFilters( +export function matchPathFilters( filters: { paths?: string[]; ignorePaths?: string[] }, filenames: string[] | undefined ): boolean { @@ -125,7 +125,7 @@ function matchPathFilters( return true; } -function matchPullRequestState(trigger: ResolvedTrigger, context: EventContext): boolean { +export function matchPullRequestState(trigger: ResolvedTrigger, context: EventContext): boolean { const labels = context.pullRequest?.labels ?? []; const labelMatches = trigger.labels !== undefined && diff --git a/packages/warden/src/types/index.ts b/packages/warden/src/types/index.ts index 828cad552..0514df2c9 100644 --- a/packages/warden/src/types/index.ts +++ b/packages/warden/src/types/index.ts @@ -115,6 +115,8 @@ export type SourceSnippet = z.infer; // Individual finding from a skill export const FindingSchema = z.object({ id: z.string(), + /** Id this finding is displayed/tracked as once dedupe matches it to an already-posted comment. Undefined means display `id` as-is. `id` itself is assigned once at creation and never mutated afterward. */ + reportedId: z.string().optional(), severity: SeveritySchema, confidence: ConfidenceSchema.optional(), title: z.string(), @@ -276,6 +278,12 @@ export const SkillErrorSchema = z.object({ }); export type SkillError = z.infer; +export const VerifierRejectionsSchema = z.object({ + count: z.number().int().nonnegative(), + reasons: z.array(z.string()), +}); +export type VerifierRejections = z.infer; + // 'analysis' = SDK/auth/abort failure, 'extraction' = parse-tier failure. export const HunkFailureSchema = z.object({ type: z.enum(['analysis', 'extraction']), @@ -349,6 +357,8 @@ export const SkillReportSchema = z.object({ traces: z.array(HunkTraceSchema).optional(), /** Set when the run cannot complete normally. */ error: SkillErrorSchema.optional(), + /** Findings the verification pass rejected, if any ran. */ + verifierRejections: VerifierRejectionsSchema.optional(), /** Usage from auxiliary LLM calls (extraction repair, semantic dedup, etc.) */ auxiliaryUsage: AuxiliaryUsageMapSchema.optional(), /** Model/runtime attribution for auxiliary LLM usage, keyed like auxiliaryUsage */ @@ -357,6 +367,8 @@ export const SkillReportSchema = z.object({ files: z.array(FileReportSchema).optional(), /** Model used for this skill's analysis */ model: z.string().optional(), + /** Distinct models observed across hunks, when they disagree */ + models: z.array(z.string()).optional(), /** Runtime backend used for this skill's analysis. */ runtime: z.string().optional(), }); diff --git a/packages/warden/src/utils/fs.test.ts b/packages/warden/src/utils/fs.test.ts new file mode 100644 index 000000000..2ce54956b --- /dev/null +++ b/packages/warden/src/utils/fs.test.ts @@ -0,0 +1,207 @@ +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import type * as NodeFS from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { writeFileAtomic, writeFilesAtomicPair } from './fs.js'; + +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, renameSync: vi.fn(actual.renameSync), writeFileSync: vi.fn(actual.writeFileSync) }; +}); + +describe('writeFileAtomic', () => { + let tempDir: string; + let targetPath: string; + + beforeEach(() => { + tempDir = join(tmpdir(), `warden-write-file-atomic-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + targetPath = join(tempDir, 'nested', 'output.json'); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it('writes the exact content and creates parent directories', () => { + writeFileAtomic(targetPath, '{"a":1}'); + expect(readFileSync(targetPath, 'utf-8')).toBe('{"a":1}'); + }); + + it('overwrites an existing file', () => { + writeFileAtomic(targetPath, 'first'); + writeFileAtomic(targetPath, 'second'); + expect(readFileSync(targetPath, 'utf-8')).toBe('second'); + }); + + it('leaves no temp file behind after repeated calls', () => { + for (let i = 0; i < 5; i++) { + writeFileAtomic(targetPath, `content-${i}`); + } + const entries = readdirSync(join(tempDir, 'nested')); + expect(entries).toEqual(['output.json']); + }); + + it('cleans up the temp file and rethrows when rename fails', async () => { + const { renameSync } = await import('node:fs'); + vi.mocked(renameSync).mockImplementationOnce(() => { + throw new Error('rename failed'); + }); + + expect(() => writeFileAtomic(targetPath, 'content')).toThrow('rename failed'); + expect(existsSync(targetPath)).toBe(false); + const entries = readdirSync(join(tempDir, 'nested')); + expect(entries).toEqual([]); + }); +}); + +describe('writeFilesAtomicPair', () => { + let tempDir: string; + let firstPath: string; + let secondPath: string; + + beforeEach(() => { + tempDir = join(tmpdir(), `warden-write-files-atomic-pair-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + firstPath = join(tempDir, 'first.json'); + secondPath = join(tempDir, 'second.json'); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + it('writes every file when all staging writes succeed', () => { + writeFilesAtomicPair([ + { path: firstPath, content: 'first-content' }, + { path: secondPath, content: 'second-content' }, + ]); + + expect(readFileSync(firstPath, 'utf-8')).toBe('first-content'); + expect(readFileSync(secondPath, 'utf-8')).toBe('second-content'); + }); + + it('leaves all target files untouched when the second file fails to stage', async () => { + writeFilesAtomicPair([{ path: firstPath, content: 'stale-first' }]); + + const actualFs = await vi.importActual('node:fs'); + const { writeFileSync } = await import('node:fs'); + vi.mocked(writeFileSync) + .mockImplementationOnce(actualFs.writeFileSync) + .mockImplementationOnce(() => { + throw new Error('disk full'); + }); + + expect(() => + writeFilesAtomicPair([ + { path: firstPath, content: 'new-first' }, + { path: secondPath, content: 'new-second' }, + ]) + ).toThrow('disk full'); + + expect(readFileSync(firstPath, 'utf-8')).toBe('stale-first'); + expect(existsSync(secondPath)).toBe(false); + }); + + it('cleans up the first file\'s staged temp file when the second file fails to stage', async () => { + const actualFs = await vi.importActual('node:fs'); + const { writeFileSync } = await import('node:fs'); + vi.mocked(writeFileSync) + .mockImplementationOnce(actualFs.writeFileSync) + .mockImplementationOnce(() => { + throw new Error('disk full'); + }); + + expect(() => + writeFilesAtomicPair([ + { path: firstPath, content: 'first-content' }, + { path: secondPath, content: 'second-content' }, + ]) + ).toThrow('disk full'); + + expect(readdirSync(tempDir)).toEqual([]); + }); + + it('restores both files to their prior content when the second file\'s commit rename fails', async () => { + writeFilesAtomicPair([ + { path: firstPath, content: 'stale-first' }, + { path: secondPath, content: 'stale-second' }, + ]); + + const { renameSync } = await import('node:fs'); + const actualFs = await vi.importActual('node:fs'); + // Both destinations already exist, so each file's commit is a backup rename + // followed by a commit rename: [first-backup, first-commit, second-backup, second-commit]. + // Fail only the fourth call (second file's commit rename). + let call = 0; + vi.mocked(renameSync).mockImplementation((...args) => { + call += 1; + if (call === 4) throw new Error('rename failed'); + return actualFs.renameSync(...args); + }); + + expect(() => + writeFilesAtomicPair([ + { path: firstPath, content: 'new-first' }, + { path: secondPath, content: 'new-second' }, + ]) + ).toThrow('rename failed'); + + expect(readFileSync(firstPath, 'utf-8')).toBe('stale-first'); + expect(readFileSync(secondPath, 'utf-8')).toBe('stale-second'); + expect(readdirSync(tempDir).sort()).toEqual(['first.json', 'second.json']); + }); + + it('restores the first file\'s prior content when its own commit rename fails, without touching the second file', async () => { + writeFilesAtomicPair([ + { path: firstPath, content: 'stale-first' }, + { path: secondPath, content: 'stale-second' }, + ]); + + const { renameSync } = await import('node:fs'); + const actualFs = await vi.importActual('node:fs'); + // First file's own backup rename succeeds, but its commit rename fails — + // exercises restoring an entry's backup before it's ever added to `committed`. + let call = 0; + vi.mocked(renameSync).mockImplementation((...args) => { + call += 1; + if (call === 2) throw new Error('rename failed'); + return actualFs.renameSync(...args); + }); + + expect(() => + writeFilesAtomicPair([ + { path: firstPath, content: 'new-first' }, + { path: secondPath, content: 'new-second' }, + ]) + ).toThrow('rename failed'); + + expect(readFileSync(firstPath, 'utf-8')).toBe('stale-first'); + expect(readFileSync(secondPath, 'utf-8')).toBe('stale-second'); + expect(readdirSync(tempDir).sort()).toEqual(['first.json', 'second.json']); + }); + + it('removes a newly-created file when its partner\'s rename fails and neither existed before', async () => { + const { renameSync } = await import('node:fs'); + const actualFs = await vi.importActual('node:fs'); + vi.mocked(renameSync) + .mockImplementationOnce(actualFs.renameSync) // first file's commit rename + .mockImplementationOnce(() => { + throw new Error('rename failed'); + }); // second file's commit rename + + expect(() => + writeFilesAtomicPair([ + { path: firstPath, content: 'new-first' }, + { path: secondPath, content: 'new-second' }, + ]) + ).toThrow('rename failed'); + + expect(existsSync(firstPath)).toBe(false); + expect(existsSync(secondPath)).toBe(false); + expect(readdirSync(tempDir)).toEqual([]); + }); +}); diff --git a/packages/warden/src/utils/fs.ts b/packages/warden/src/utils/fs.ts new file mode 100644 index 000000000..15fa0f7d6 --- /dev/null +++ b/packages/warden/src/utils/fs.ts @@ -0,0 +1,86 @@ +import { existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; +import { dirname } from 'node:path'; + +/** + * Write a file's full contents atomically: write to a temp path in the same + * directory, then rename into place. Readers never observe a partial write. + * On failure, best-effort removes the temp file before rethrowing. + */ +export function writeFileAtomic(path: string, content: string): void { + mkdirSync(dirname(path), { recursive: true }); + const tempPath = `${path}.${process.pid}.${Date.now()}.tmp`; + try { + writeFileSync(tempPath, content); + renameSync(tempPath, path); + } catch (error) { + try { unlinkSync(tempPath); } catch { /* best-effort cleanup */ } + throw error; + } +} + +/** + * Write multiple files as one atomic unit: stage every file's content to a + * temp path first, then rename each temp file into place, backing up + * whichever destination already existed. If a staging write fails, none of + * the target paths are touched. If a rename fails partway through the commit + * phase, every destination already renamed this call is restored from its + * backup (or removed, if it didn't exist before) before rethrowing. Either + * way, callers that require several files to stay in sync (e.g. a + * metadata/findings pair sharing a runId) never observe a partial commit — + * one file updated to a new run while its partner still holds a prior one. + */ +export function writeFilesAtomicPair(files: { path: string; content: string }[]): void { + const staged: { tempPath: string; path: string }[] = []; + try { + for (const { path, content } of files) { + mkdirSync(dirname(path), { recursive: true }); + const tempPath = `${path}.${process.pid}.${Date.now()}.tmp`; + writeFileSync(tempPath, content); + staged.push({ tempPath, path }); + } + } catch (error) { + for (const { tempPath } of staged) { + try { unlinkSync(tempPath); } catch { /* best-effort cleanup */ } + } + throw error; + } + + const committed: { path: string; backupPath?: string }[] = []; + try { + for (const { tempPath, path } of staged) { + const backupPath = existsSync(path) ? `${path}.${process.pid}.${Date.now()}.bak` : undefined; + if (backupPath) renameSync(path, backupPath); + try { + renameSync(tempPath, path); + } catch (error) { + // The backup rename above (if any) already succeeded for this entry, so it isn't + // in `committed` yet — restore it here before letting the outer catch handle the + // entries from earlier iterations that did make it into `committed`. + if (backupPath) { + try { renameSync(backupPath, path); } catch { /* best-effort rollback */ } + } + throw error; + } + committed.push({ path, backupPath }); + } + } catch (error) { + for (const { path, backupPath } of committed) { + try { + if (backupPath) renameSync(backupPath, path); + else unlinkSync(path); + } catch { /* best-effort rollback */ } + } + for (const { tempPath, path } of staged) { + if (!committed.some((entry) => entry.path === path)) { + try { unlinkSync(tempPath); } catch { /* best-effort cleanup */ } + } + } + throw error; + } + + for (const { backupPath } of committed) { + if (backupPath) { + try { unlinkSync(backupPath); } catch { /* best-effort cleanup */ } + } + } +} diff --git a/packages/warden/src/utils/index.ts b/packages/warden/src/utils/index.ts index 4b57e832e..a5f8db055 100644 --- a/packages/warden/src/utils/index.ts +++ b/packages/warden/src/utils/index.ts @@ -8,6 +8,7 @@ export { GIT_NON_INTERACTIVE_ENV, } from './exec.js'; export { isPathLike } from './path.js'; +export { writeFileAtomic } from './fs.js'; export type { ExecOptions } from './exec.js'; /** Default concurrency for parallel trigger/skill execution */ diff --git a/packages/warden/src/utils/version.test.ts b/packages/warden/src/utils/version.test.ts new file mode 100644 index 000000000..587450299 --- /dev/null +++ b/packages/warden/src/utils/version.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest'; + +describe('getVersion', () => { + beforeEach(() => { + vi.resetModules(); + }); + + function occurrencesOf(needle: string, haystack: string): number { + return haystack.split(needle).length - 1; + } + + it('reads the package.json two levels up from the compiled file', async () => { + vi.doMock('node:fs', () => ({ + existsSync: (path: string) => occurrencesOf('packages/warden', path) <= 1, + readFileSync: () => JSON.stringify({ name: '@sentry/warden', version: '1.2.3' }), + })); + const { getVersion } = await import('./version.js'); + expect(getVersion()).toBe('1.2.3'); + }); + + it('falls back to packages/warden/package.json when the two-levels-up file has no version (ncc bundle layout)', async () => { + vi.doMock('node:fs', () => ({ + existsSync: (path: string) => occurrencesOf('packages/warden', path) === 2, + readFileSync: (path: string) => + occurrencesOf('packages/warden', path) === 2 + ? JSON.stringify({ name: '@sentry/warden', version: '0.42.0' }) + : JSON.stringify({ name: 'warden-monorepo' }), + })); + const { getVersion } = await import('./version.js'); + expect(getVersion()).toBe('0.42.0'); + }); + + it('ignores a version field on a package.json that is not @sentry/warden (e.g. the monorepo root)', async () => { + vi.doMock('node:fs', () => ({ + existsSync: () => true, + readFileSync: (path: string) => + occurrencesOf('packages/warden', path) === 2 + ? JSON.stringify({ name: '@sentry/warden', version: '0.42.0' }) + : JSON.stringify({ name: 'warden-monorepo', version: '1.0.0' }), + })); + const { getVersion } = await import('./version.js'); + expect(getVersion()).toBe('0.42.0'); + }); + + it('falls back to 0.0.0 when no package.json with a version is found', async () => { + vi.doMock('node:fs', () => ({ + existsSync: () => false, + readFileSync: () => '{}', + })); + const { getVersion } = await import('./version.js'); + expect(getVersion()).toBe('0.0.0'); + }); +}); diff --git a/packages/warden/src/utils/version.ts b/packages/warden/src/utils/version.ts index b9bdfc23a..28ce9985b 100644 --- a/packages/warden/src/utils/version.ts +++ b/packages/warden/src/utils/version.ts @@ -1,14 +1,26 @@ -import { readFileSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; let cachedVersion: string | undefined; +/** Only trusts a version from the actual @sentry/warden package.json, not just any package.json that happens to exist at a candidate path. */ +function readPackageVersion(path: string): string | undefined { + if (!existsSync(path)) return undefined; + const pkg = JSON.parse(readFileSync(path, 'utf-8')) as { name?: string; version?: string }; + return pkg.name === '@sentry/warden' ? pkg.version : undefined; +} + export function getVersion(): string { if (cachedVersion) return cachedVersion; const __dirname = dirname(fileURLToPath(import.meta.url)); - const pkg = JSON.parse(readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf-8')) as { version: string }; - cachedVersion = pkg.version; + // Normal build: dist//utils/version.js, two levels below the + // package root. ncc-bundled action: dist/action/index.js at the monorepo root, + // where packages/warden/package.json is a sibling rather than an ancestor. + cachedVersion = + readPackageVersion(join(__dirname, '..', '..', 'package.json')) ?? + readPackageVersion(join(__dirname, '..', '..', 'packages', 'warden', 'package.json')) ?? + '0.0.0'; return cachedVersion; } diff --git a/specs/jsonl-schema.json b/specs/jsonl-schema.json index 6c5cf233a..750e6449d 100644 --- a/specs/jsonl-schema.json +++ b/specs/jsonl-schema.json @@ -94,6 +94,9 @@ }, "trace": { "$ref": "#/$defs/HunkTrace" + }, + "verifierRejections": { + "$ref": "#/$defs/__schema6" } }, "required": [ @@ -149,6 +152,9 @@ "id": { "type": "string" }, + "reportedId": { + "type": "string" + }, "severity": { "type": "string", "enum": [ @@ -629,6 +635,27 @@ ], "additionalProperties": false }, + "__schema6": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reasons": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "count", + "reasons" + ], + "additionalProperties": false + }, "SkillRecord": { "type": "object", "properties": { @@ -689,6 +716,9 @@ "error": { "$ref": "#/$defs/SkillError" }, + "verifierRejections": { + "$ref": "#/$defs/__schema6" + }, "auxiliaryUsage": { "$ref": "#/$defs/AuxiliaryUsage" }, @@ -725,6 +755,12 @@ "model": { "type": "string" }, + "models": { + "type": "array", + "items": { + "type": "string" + } + }, "runtime": { "type": "string" }, @@ -882,6 +918,11 @@ "minimum": 0, "maximum": 9007199254740991 }, + "totalVerifierRejections": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, "error": { "$ref": "#/$defs/SkillError" } diff --git a/specs/output-schema-v2-migration.md b/specs/output-schema-v2-migration.md new file mode 100644 index 000000000..11cf5dcd6 --- /dev/null +++ b/specs/output-schema-v2-migration.md @@ -0,0 +1,128 @@ +# Output Schema v2 Migration Guide + +Warden's findings-file output has a new, opt-in schema (`output-schema-version: '2'` +on the action, default `'1'`). This guide maps every v1 field to its v2 +equivalent for downstream consumers writing their own ingestion. + +## Why + +Two structural gaps in v1, raised in maintainer conversation about splitting a +"metadata" file out of the findings-centered file: + +1. **Auxiliary-model rewrites were invisible per finding.** A verification pass + can rewrite a finding's title/description/severity/confidence using a + different model than the one that produced it, and a merge pass can fold + sibling findings together. v1 tracked this only in aggregate, at the + `SkillReport` level (`auxiliaryUsage`/`auxiliaryUsageAttribution`) — a + reader couldn't tell, for one specific finding, whether it was rewritten, + by what model, or what it looked like before. +2. **Cross-skill attribution only existed as GitHub comment text.** When + multiple skills flag the same issue, the poster already tracks this + (`ExistingComment.skills`) and renders "Identified by Warden skillA, + skillB" into the PR comment — but that was invisible in the JSON, which + nests each finding under whichever single skill's report happened to + contain it. + +Two concrete downstream pain points as well: consumers were hand-recomputing +per-skill severity breakdowns because only a global summary existed, and +hand-maintaining a harness-version constant because the output carried no +version identity. + +## File split + +Two files, joined by `runId` (+ `runAttempt` for re-runs). A reader should +treat a `runId`/`schemaVersion` mismatch between the two as a hard error. + +- **`warden-metadata.json`** — static run context that isn't itself a skill + result or a finding: repo/PR identity, harness version, resolved run-wide + config, which skills/triggers were configured and whether they fired, and + why any were skipped. +- **`warden-findings-v2.json`** — everything that's a direct, detailed record of + a skill executing and what it produced: skill executions (model, duration, + cost, severity breakdown, errors — these are results, not context) and the + findings themselves. + +## Field mapping + +### Envelope / identity + +| v1 (`warden-findings.json`) | v2 | Notes | +|---|---|---| +| `version: '1'` | `schemaVersion: '2'` (both files) | | +| `timestamp` | `metadata.generatedAt` | | +| `runId` | `metadata.runId` and `findings.runId` | join key | +| — | `metadata.runAttempt` | new | +| — | `metadata.harness.name/version/actionRef` | new — `version` replaces hand-maintained `HARNESS_VERSION` env vars | +| `repository` | `metadata.repository` | unchanged shape | +| `event` | `metadata.event` | unchanged | +| `pullRequest` | `metadata.pullRequest` | unchanged shape | + +### Skill execution stats + +| v1 | v2 | Notes | +|---|---|---| +| `skills[]` | `findings.skillExecutions[]` | now keyed by `skillExecutionId`, not array position — a skill with multiple triggers (e.g. different models per action) gets one row per execution, disambiguated | +| `skills[].name` | `skillExecutions[].skillName` | | +| — | `skillExecutions[].skillExecutionId`, `.triggerId`, `.triggerName` | new — stable identity per skill×trigger | +| `skills[].model` | `skillExecutions[].model` | value now reflects the model that actually answered (from the live API response), not just the configured override — see `metadata.resolvedDefaults.model` for the configured value; the two intentionally diverge when no model override was set or hunks didn't all agree | +| — | `skillExecutions[].models` | new — full set of distinct observed models across this execution's hunks, present only when they disagree (otherwise collapsed into `.model`) | +| — | `skillExecutions[].runtime`, `.auxiliaryModel`, `.synthesisModel` | new | +| `skills[].durationMs` | `skillExecutions[].durationMs` | | +| `skills[].usage` | `skillExecutions[].usage` | unchanged shape | +| — | `skillExecutions[].auxiliaryUsage[]` | new — array of `{agent, model, runtime, usage}`, replacing the record-keyed `SkillReport.auxiliaryUsage`/`auxiliaryUsageAttribution` maps for export purposes | +| `skills[].findings.length` (recomputed) | `skillExecutions[].findingIds.length` | | +| *(recomputed by hand from `skills[].findings[]`)* | `skillExecutions[].findingsBySeverity` | **fixed** — precomputed, no more manual recompute | +| `skills[].verifierRejections` | `skillExecutions[].verifierRejections` | unchanged shape | +| `skills[].failedHunks/.failedExtractions/.error` | `skillExecutions[].failedHunks/.failedExtractions/.error` | unchanged shape | + +### Findings + +| v1 | v2 | Notes | +|---|---|---| +| `skills[].findings[]` (nested per skill) | `findings[]` (flat, top-level) | a finding is no longer owned by exactly one skill array | +| `findings[].id/.severity/.confidence/.title/.description/.location/.additionalLocations/.sourceSnippet` | same fields on `findings[]` | unchanged | +| — | `findings[].contentHash` | new — stable cross-run key (same as `dedup.ts`'s `generateContentHash`) | +| — | `findings[].reportedId` | new, optional — set once dedupe matches this finding to an existing GitHub comment, for continuity with the id shown across runs; `id` itself never changes | +| *(dropped from export)* | `findings[].verification` | **fixed** — the verifier's evidence text was already rendered into GitHub comments but silently absent from v1 JSON | +| *(implicit, one skill only)* | `findings[].reportedBy[]` | new — `{skillExecutionId, skillName, role: 'primary'\|'corroborating', matchType?}`; a finding independently flagged by multiple skills now lists all of them | +| *(only in `auxiliaryUsage` aggregate)* | `findings[].provenance.verification` | new — `{outcome: 'kept'\|'revised', model, runtime, evidence?, before?}`; on `revised`, `before` holds the pre-revision title/description/severity/confidence | +| *(only in `auxiliaryUsage` aggregate)* | `findings[].provenance.merge` | new — `{model, runtime, absorbedFindingIds[]}` | +| *(not exported)* | `discardedFindings[]` | new, optional (omitted when empty) — verifier-rejected and merge-absorbed candidates that never reached `findings[]`, each with `{originSkillExecutionId, stage, severity, title, location?, model?, reason?, survivorFindingId?}` | + +### Observability + +| v1 | v2 | Notes | +|---|---|---| +| `configuredSkills[]` | `metadata.configuredSkills[]` | moved to metadata file, same shape | +| `triggerResults[]` | `metadata.triggerResults[]` | moved to metadata file; the embedded `report` field is dropped (that content now lives in `findings.skillExecutions`/`findings`) | +| *(not tracked)* | `metadata.skippedTriggers[]` | new — `{skillName, triggerId?, triggerName?, reason}`, reason is one of `no_event_match\|path_filter\|draft_state\|label_mismatch\|no_changes\|pending` (`pending` only appears while a scheduled run's triggers are still being worked through sequentially) | +| `findingObservations[].skill` | `findingObservations[].origin.skillExecutionId/.skillName` | renamed/restructured | +| `findingObservations[].finding` | `findingObservations[].finding` | narrowed — only `id/.reportedId/.severity/.confidence/.title/.description/.location/.elapsedMs`; `additionalLocations`/`sourceSnippet`/`verification` are dropped here (they're still exported on live findings via top-level `findings[]`, but a finding that's `deduped`/`skipped`/`resolved`/`failed` and never reaches `findings[]` loses them entirely) | +| `findingObservations[].dedupe` | `findingObservations[].dedupe` | unchanged, plus new `existingSkills[]` and `existingSkillExecutionId` fields (cross-skill attribution at the moment of the match) | +| `summary.totalFindings/.findingsBySeverity/.totalSkills` | `findings.summary.totalFindings/.bySeverity/.totalSkillExecutions` + `.byOutcome` | `totalSkillExecutions` replaces `totalSkills` since one skill can now have multiple executions; `byOutcome` is new | + +## Two-phase `analyze`/`report` mode + +v1's `TriggerRunResultSchema.report` (embedded in `triggerResults[]`) let +`report` mode replay a prior `analyze` run's output across a job boundary. In +v2 this content moves to `findings.skillExecutions`/`findings`, so `report` +mode reconstructs the same data by reading **both** v2 files and joining +`skillExecutions[]` (by `triggerId`) with `findings[]` (via `findingIds`) +instead of reading the embedded `report`. + +## Rollout + +`output-schema-version` defaults to `'1'`. Setting it to `'2'` writes both +`warden-metadata.json` and `warden-findings-v2.json` in addition to the v1 +`warden-findings.json` (v1's filename is unchanged; nothing currently +depending on it needs to change). There is no plan to flip the default +without a separate, explicitly announced deprecation window. + +## Live output + +The metadata/findings pair is also written incrementally as each trigger +completes, not only once at the end of the run — see `specs/reporters.md`, +"Schema-v2 Live Output" for the write-timing contract, the `.done` completion +marker, and the `warden runs follow ` command that tails +it. This is aimed at local development of Warden's own Action workflow code, +not a production CI capability. diff --git a/specs/reporters.md b/specs/reporters.md index 99149e8ef..49a8579c6 100644 --- a/specs/reporters.md +++ b/specs/reporters.md @@ -843,3 +843,15 @@ findings-count=5 high-count=2 summary=Warden found 5 issues across 2 skills ``` + +### Schema-v2 Live Output + +When `output-schema-version: '2'`, the `warden-metadata.json`/`warden-findings-v2.json` pair (see [`specs/output-schema-v2-migration.md`](./output-schema-v2-migration.md)) is written incrementally as each trigger completes, not only once at the end of the run. This mirrors the JSONL live-append behavior in Section 3, at the granularity schema v2 actually supports: per-trigger completion, since a `skillExecutions`/`findings` entry only exists once a full skill report does — there is no chunk-level record in schema v2. + +**Write timing.** Every trigger completion (success or error) triggers a full atomic rewrite of both files, reflecting only the trigger results known at that point. Posting-derived data — `findingObservations`, per-finding `githubCommentId`/`githubCommentUrl`, and `reportedBy` corroboration beyond the primary attribution — is legitimately absent from every intermediate write; it only appears once the existing post-analysis rewrite runs after comment posting completes. This is unchanged by live output. + +**Completion marker.** A `.done` sidecar (`{findingsPath}.done`) is written next to `warden-findings-v2.json` only at the run's one true final write. Its absence is the signal a follower uses to know the run is still in progress, the same role it plays for JSONL logs in Section 3. + +**`pending` skip reason.** `schedule.ts` builds its matched-trigger list incrementally inside a sequential loop, so an intermediate write may include triggers this loop has not reached yet. Those report `skippedTriggers[].reason: 'pending'` rather than a real skip reason, distinguishing "not evaluated yet" from a genuine skip (`no_event_match`, `path_filter`, etc.). `pr-workflow.ts`'s trigger list is fully resolved before any trigger runs, so `'pending'` never appears there. + +**Consuming it.** `warden runs follow ` tails the pair the same way `warden runs follow` tails a JSONL log — re-reading and re-parsing both files on each change (there is no append semantics for a rewritten JSON document), rendering each newly-completed skill execution, and stopping once the `.done` sidecar appears. This is intended for local development of the Action's own workflow code (running `pr-workflow.ts`/`schedule.ts` against a local checkout with a second terminal following along), not as a production CI capability — a GitHub-hosted Action run is a single synchronous process with nothing external able to follow its files during the run itself.