test(github): Add push permission outcome evals

packages/junior-evals/evals/github/skill-workflows.eval.ts

@@ -52,6 +52,79 @@ describeEval("GitHub Skill Workflows", slackEvals, (it) => {
     expect(verifyCall).toMatchObject({ result: { ok: true } });
   });
 
+  it("when an earlier push is denied, reconcile before retrying and report the successful remote state", async ({
+    run,
+  }) => {
+    const result = await run({
+      overrides: {
+        skill_dirs: ["fixtures/github-push-outcomes-skills"],
+      },
+      initialEvents: [
+        mention(
+          "Ship the release-status change using the local `earlier-denial` push fixture. Complete the work and tell me the final remote state.",
+        ),
+      ],
+      criteria: rubric({
+        pass: [
+          "The assistant reconciles remote state after the simulated HTTP 403 before retrying.",
+          "After verification shows the first attempt did not apply the mutation, the assistant retries once and reports that the remote contains the shipped status.",
+        ],
+        fail: [
+          "Do not ask for OAuth, a token, or changed GitHub permissions for the local fixture.",
+          "Do not report the earlier permission denial as the final outcome after verification proves the retry succeeded.",
+          "Do not contact production GitHub.",
+        ],
+      }),
+    });
+
+    const reconciledResult = toolCalls(result.session).find(
+      (call) =>
+        call.name === "bash" &&
+        call.status === "ok" &&
+        JSON.stringify(call.result)?.includes(
+          "reconciled_remote_status=present push_attempts=2",
+        ) === true,
+    );
+    expect(reconciledResult).toMatchObject({ result: { ok: true } });
+  });
+
+  it("when a stale denial follows an applied push, reconcile and do not repeat the mutation", async ({
+    run,
+  }) => {
+    const result = await run({
+      overrides: {
+        skill_dirs: ["fixtures/github-push-outcomes-skills"],
+      },
+      initialEvents: [
+        mention(
+          "Ship the release-status change using the local `denial-after-apply` push fixture. Complete the work and tell me the final remote state.",
+        ),
+      ],
+      criteria: rubric({
+        pass: [
+          "The assistant treats the simulated HTTP 403 as inconclusive until it verifies remote state.",
+          "After verification proves the mutation is present remotely, the assistant reports success despite the stale denial result.",
+        ],
+        fail: [
+          "Do not retry the mutation after verification proves it already applied.",
+          "Do not ask for OAuth, a token, or changed GitHub permissions for the local fixture.",
+          "Do not report the stale permission denial as the final outcome.",
+          "Do not contact production GitHub.",
+        ],
+      }),
+    });
+
+    const reconciledResult = toolCalls(result.session).find(
+      (call) =>
+        call.name === "bash" &&
+        call.status === "ok" &&
+        JSON.stringify(call.result)?.includes(
+          "reconciled_remote_status=present push_attempts=1",
+        ) === true,
+    );
+    expect(reconciledResult).toMatchObject({ result: { ok: true } });
+  });
+
   it("when asked about PR auth sequencing, explain automatic installation credentials", async ({
     run,
   }) => {

packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/SKILL.md

@@ -0,0 +1,24 @@
+---
+name: github-push-outcomes-fixture
+description: Use for requests involving the local simulated GitHub push outcome fixture.
+---
+
+# GitHub Push Outcomes Fixture
+
+The fixture is at `skills/github-push-outcomes-fixture` and never contacts GitHub.
+
+Run `setup.sh <scenario>` before changing `project/src/status.ts` from `pending` to
+`shipped` and committing it. Push with `push.sh <scenario>`, then reconcile the
+result with `verify.sh` before deciding what happened or whether to retry.
+
+Scenarios:
+
+- `earlier-denial`: the first push returns a genuine simulated HTTP 403 without
+  applying the remote mutation. After verification proves the mutation absent,
+  one retry succeeds.
+- `denial-after-apply`: the push applies the simulated remote mutation but still
+  returns a stale HTTP 403 result. Verification proves the mutation present; do
+  not retry it or ask for permissions.
+
+Report the reconciled remote state, not merely the last error text. Never request
+OAuth, tokens, or permission changes for this local fixture.

packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/project/src/status.ts

@@ -0,0 +1 @@
+export const releaseStatus = "pending";

packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/push.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -eu
+
+scenario=${1:?scenario required}
+case "$scenario" in
+  earlier-denial|denial-after-apply) ;;
+  *) printf 'unknown scenario: %s\n' "$scenario" >&2; exit 2 ;;
+esac
+
+root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+state="$root/.state"
+project="$root/project"
+status=$(git -C "$project" show HEAD:src/status.ts)
+if [ "$status" != 'export const releaseStatus = "shipped";' ]; then
+  printf 'local commit does not contain shipped status\n' >&2
+  exit 2
+fi
+
+attempts=$(cat "$state/attempts")
+attempts=$((attempts + 1))
+printf '%s\n' "$attempts" > "$state/attempts"
+
+case "$scenario" in
+  earlier-denial)
+    if [ "$attempts" -eq 1 ]; then
+      printf 'remote: HTTP 403: Resource not accessible by integration (simulated earlier denial)\n' >&2
+      exit 1
+    fi
+    printf 'present\n' > "$state/remote-status"
+    printf 'simulated push accepted on retry\n'
+    ;;
+  denial-after-apply)
+    printf 'present\n' > "$state/remote-status"
+    printf 'remote: HTTP 403: stale permission result after simulated remote accepted mutation\n' >&2
+    exit 1
+    ;;
+esac

packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/setup.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -eu
+
+scenario=${1:?scenario required}
+case "$scenario" in
+  earlier-denial|denial-after-apply) ;;
+  *) printf 'unknown scenario: %s\n' "$scenario" >&2; exit 2 ;;
+esac
+
+root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+project="$root/project"
+state="$root/.state"
+rm -rf "$state" "$project/.git"
+mkdir -p "$state"
+printf '0\n' > "$state/attempts"
+printf 'absent\n' > "$state/remote-status"
+
+git -C "$project" init -b junior/push-outcome
+git -C "$project" config user.name "Junior Eval"
+git -C "$project" config user.email "junior-eval@example.com"
+printf 'export const releaseStatus = "pending";\n' > "$project/src/status.ts"
+git -C "$project" add src/status.ts
+git -C "$project" commit -m "Add pending release status"
+printf 'initialized %s push fixture\n' "$scenario"

packages/junior-evals/fixtures/github-push-outcomes-skills/github-push-outcomes-fixture/verify.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -eu
+
+root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+state="$root/.state"
+status=$(cat "$state/remote-status")
+attempts=$(cat "$state/attempts")
+printf 'reconciled_remote_status=%s push_attempts=%s\n' "$status" "$attempts"
+[ "$status" = "present" ]