diff --git a/packages/junior-evals/evals/github/skill-workflows.eval.ts b/packages/junior-evals/evals/github/skill-workflows.eval.ts index 14cde2bd6..5a8301380 100644 --- a/packages/junior-evals/evals/github/skill-workflows.eval.ts +++ b/packages/junior-evals/evals/github/skill-workflows.eval.ts @@ -52,6 +52,79 @@ describeEval("GitHub Skill Workflows", slackEvals, (it) => { expect(verifyCall).toMatchObject({ result: { ok: true } }); }); + it("when an earlier push is denied, reconcile before retrying and report the successful remote state", async ({ + run, + }) => { + const result = await run({ + overrides: { + skill_dirs: ["fixtures/github-push-outcomes-skills"], + }, + initialEvents: [ + mention( + "Ship the release-status change using the local `earlier-denial` push fixture. Complete the work and tell me the final remote state.", + ), + ], + criteria: rubric({ + pass: [ + "The assistant reconciles remote state after the simulated HTTP 403 before retrying.", + "After verification shows the first attempt did not apply the mutation, the assistant retries once and reports that the remote contains the shipped status.", + ], + fail: [ + "Do not ask for OAuth, a token, or changed GitHub permissions for the local fixture.", + "Do not report the earlier permission denial as the final outcome after verification proves the retry succeeded.", + "Do not contact production GitHub.", + ], + }), + }); + + const reconciledResult = toolCalls(result.session).find( + (call) => + call.name === "bash" && + call.status === "ok" && + JSON.stringify(call.result)?.includes( + "reconciled_remote_status=present push_attempts=2", + ) === true, + ); + expect(reconciledResult).toMatchObject({ result: { ok: true } }); + }); + + it("when a stale denial follows an applied push, reconcile and do not repeat the mutation", async ({ + run, + }) => { + const result = await run({ + overrides: { + skill_dirs: ["fixtures/github-push-outcomes-skills"], + }, + initialEvents: [ + mention( + "Ship the release-status change using the local `denial-after-apply` push fixture. Complete the work and tell me the final remote state.", + ), + ], + criteria: rubric({ + pass: [ + "The assistant treats the simulated HTTP 403 as inconclusive until it verifies remote state.", + "After verification proves the mutation is present remotely, the assistant reports success despite the stale denial result.", + ], + fail: [ + "Do not retry the mutation after verification proves it already applied.", + "Do not ask for OAuth, a token, or changed GitHub permissions for the local fixture.", + "Do not report the stale permission denial as the final outcome.", + "Do not contact production GitHub.", + ], + }), + }); + + const reconciledResult = toolCalls(result.session).find( + (call) => + call.name === "bash" && + call.status === "ok" && + JSON.stringify(call.result)?.includes( + "reconciled_remote_status=present push_attempts=1", + ) === true, + ); + expect(reconciledResult).toMatchObject({ result: { ok: true } }); + }); + it("when asked about PR auth sequencing, explain automatic installation credentials", async ({ run, }) => { diff --git a/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/SKILL.md b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/SKILL.md new file mode 100644 index 000000000..b01a19a88 --- /dev/null +++ b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/SKILL.md @@ -0,0 +1,24 @@ +--- +name: github-push-outcomes-fixture +description: Use for requests involving the local simulated GitHub push outcome fixture. +--- + +# GitHub Push Outcomes Fixture + +The fixture is at `skills/github-push-outcomes-fixture` and never contacts GitHub. + +Run `setup.sh ` before changing `project/src/status.ts` from `pending` to +`shipped` and committing it. Push with `push.sh `, then reconcile the +result with `verify.sh` before deciding what happened or whether to retry. + +Scenarios: + +- `earlier-denial`: the first push returns a genuine simulated HTTP 403 without + applying the remote mutation. After verification proves the mutation absent, + one retry succeeds. +- `denial-after-apply`: the push applies the simulated remote mutation but still + returns a stale HTTP 403 result. Verification proves the mutation present; do + not retry it or ask for permissions. + +Report the reconciled remote state, not merely the last error text. Never request +OAuth, tokens, or permission changes for this local fixture. diff --git a/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/project/src/status.ts b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/project/src/status.ts new file mode 100644 index 000000000..5bb996d4b --- /dev/null +++ b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/project/src/status.ts @@ -0,0 +1 @@ +export const releaseStatus = "pending"; diff --git a/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/push.sh b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/push.sh new file mode 100755 index 000000000..5f1340ece --- /dev/null +++ b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/push.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -eu + +scenario=${1:?scenario required} +case "$scenario" in + earlier-denial|denial-after-apply) ;; + *) printf 'unknown scenario: %s\n' "$scenario" >&2; exit 2 ;; +esac + +root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +state="$root/.state" +project="$root/project" +status=$(git -C "$project" show HEAD:src/status.ts) +if [ "$status" != 'export const releaseStatus = "shipped";' ]; then + printf 'local commit does not contain shipped status\n' >&2 + exit 2 +fi + +attempts=$(cat "$state/attempts") +attempts=$((attempts + 1)) +printf '%s\n' "$attempts" > "$state/attempts" + +case "$scenario" in + earlier-denial) + if [ "$attempts" -eq 1 ]; then + printf 'remote: HTTP 403: Resource not accessible by integration (simulated earlier denial)\n' >&2 + exit 1 + fi + printf 'present\n' > "$state/remote-status" + printf 'simulated push accepted on retry\n' + ;; + denial-after-apply) + printf 'present\n' > "$state/remote-status" + printf 'remote: HTTP 403: stale permission result after simulated remote accepted mutation\n' >&2 + exit 1 + ;; +esac diff --git a/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/setup.sh b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/setup.sh new file mode 100755 index 000000000..a159316ff --- /dev/null +++ b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/setup.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -eu + +scenario=${1:?scenario required} +case "$scenario" in + earlier-denial|denial-after-apply) ;; + *) printf 'unknown scenario: %s\n' "$scenario" >&2; exit 2 ;; +esac + +root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +project="$root/project" +state="$root/.state" +rm -rf "$state" "$project/.git" +mkdir -p "$state" +printf '0\n' > "$state/attempts" +printf 'absent\n' > "$state/remote-status" + +git -C "$project" init -b junior/push-outcome +git -C "$project" config user.name "Junior Eval" +git -C "$project" config user.email "junior-eval@example.com" +printf 'export const releaseStatus = "pending";\n' > "$project/src/status.ts" +git -C "$project" add src/status.ts +git -C "$project" commit -m "Add pending release status" +printf 'initialized %s push fixture\n' "$scenario" diff --git a/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/verify.sh b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/verify.sh new file mode 100755 index 000000000..cd6c1c12a --- /dev/null +++ b/packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/verify.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eu + +root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +state="$root/.state" +status=$(cat "$state/remote-status") +attempts=$(cat "$state/attempts") +printf 'reconciled_remote_status=%s push_attempts=%s\n' "$status" "$attempts" +[ "$status" = "present" ]