From f4352471c6b98e6f66dd5fb56df5d9bbe94bf878 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 4 Jul 2026 12:09:08 +0100 Subject: [PATCH 1/2] feat: adapt apply scan window Expand skip-heavy zero-close scans without changing close or worker limits, preserve close-health telemetry across comment sync, and extract apply helpers so GitHub can validate the workflow. Co-authored-by: brokemac79 --- CHANGELOG.md | 2 + docs/scheduler.md | 9 ++ scripts/apply-workflow-helpers.sh | 100 +++++++++++++++++++++ src/clawsweeper.ts | 14 ++- src/repair/workflow-utils.ts | 136 ++++++++++++++++++++++++++++- test/clawsweeper.test.ts | 3 + test/repair/workflow-utils.test.ts | 131 +++++++++++++++++++++++++++ test/sweep-workflow.test.ts | 54 +++++++++--- 8 files changed, 433 insertions(+), 16 deletions(-) create mode 100644 scripts/apply-workflow-helpers.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c6d1ae1d6..2976f1b7f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ checkpoint, and status-only commits are intentionally omitted. ### Changed +- Expanded untargeted close-apply scans from 300 toward a capped 900 records after skip-heavy zero-close windows without changing close or worker limits. Thanks @brokemac79. - Made ClawHub diversion comments a practical self-serve handoff with package-shape, manifest, configuration, documentation, usage, and smoke-proof guidance. Thanks @brokemac79. - Reduced duplicate GitHub API reads in each live-dashboard status snapshot and batched recent automerge hydration into one GraphQL request with a REST fallback. Thanks @brokemac79. - Raised the apply-existing close limit and checkpoint size from 5 to 20 fresh closes per run so continuation chains drain the proposal queue faster while each GitHub App token stays within its lifetime. @@ -26,6 +27,7 @@ checkpoint, and status-only commits are intentionally omitted. ### Fixed +- Split apply workflow helpers out of the oversized inline expression so GitHub can validate and start sweep runs again. - Bounded apply-existing checkpoints to five fresh closes, renewed the GitHub App token between continuation runs, and stopped zero-progress scans from chaining indefinitely. diff --git a/docs/scheduler.md b/docs/scheduler.md index 17de840291..674d57b72b 100644 --- a/docs/scheduler.md +++ b/docs/scheduler.md @@ -459,6 +459,15 @@ continuation with a fresh GitHub App token after any checkpoint that closes at least one item. A saturated scan that closes nothing stops without chaining so the same records cannot create an unbounded runner loop. +Untargeted cursor-based close apply starts with a 300-record scan window. If +the previous cursor window was a full close-mode scan, closed nothing, skipped +at least 80% of processed records, and did not hit a live-fetch, runtime-budget, +or missing-cursor failure, the next automatic window expands to inspect more +records, capped at 900. This changes only the deterministic scan window: +`apply_limit`, checkpoint size, close gates, live-state checks, and maintainer +policy gates stay unchanged. The workflow logs and sweep status detail include +the selected scan window and reason. + Before a close-mode apply run starts, the workflow summarizes the selected close candidate mix by quality bucket in the status detail. Buckets such as implemented-on-main, duplicate/superseded, needs PR close proof, diff --git a/scripts/apply-workflow-helpers.sh b/scripts/apply-workflow-helpers.sh new file mode 100644 index 0000000000..cd5fdf2066 --- /dev/null +++ b/scripts/apply-workflow-helpers.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +# Shared by the apply workflow step. The caller supplies the current apply +# settings as shell variables before sourcing this file. +# shellcheck disable=SC2034,SC2154 + +publish_changes() { + local message="$1" + shift + local publish_args=(--message "$message" --rebase-strategy apply-records) + local path + for path in "$@"; do + publish_args+=(--path "$path") + done + pnpm run repair:publish-main -- "${publish_args[@]}" +} + +publish_status() { + local message="$1" + if ! publish_changes "$message" results/sweep-status; then + echo "Best-effort status update failed: $message" + git restore results/sweep-status || true + fi +} + +write_apply_health() { + local report_path="$1" + local output_path="$2" + local health_mode="$3" + local health_processed_limit="$4" + local health_cursor_path="${5:-}" + local health_cursor_required="${6:-false}" + local health_candidate_count="${7:-}" + local health_scheduled_interval_minutes="${8:-}" + local health_cursor_advance_count="${9:-}" + local health_args=( + --target-repo "$TARGET_REPO" + --report "$report_path" + --mode "$health_mode" + --processed-limit "$health_processed_limit" + --close-limit "$limit" + ) + if [ -n "$health_cursor_path" ]; then + health_args+=(--cursor-path "$health_cursor_path") + fi + if [ "$health_cursor_required" = "true" ]; then + health_args+=(--cursor-required true) + fi + if [ -n "$health_candidate_count" ]; then + health_args+=(--candidate-count "$health_candidate_count") + fi + if [ -n "$health_scheduled_interval_minutes" ]; then + health_args+=(--scheduled-interval-minutes "$health_scheduled_interval_minutes") + fi + if [ -n "$health_cursor_advance_count" ]; then + health_args+=(--cursor-advance-count "$health_cursor_advance_count") + fi + pnpm run --silent workflow -- summarize-apply-report "${health_args[@]}" > "$output_path" +} + +select_adaptive_apply_batch() { + if [ "$sync_comments_only" = "true" ] || [ -n "$item_numbers" ]; then + return + fi + mkdir -p .artifacts + local adaptive_batch_env=".artifacts/apply-adaptive-batch.env" + pnpm run --silent workflow -- adaptive-apply-batch-size \ + --status-path "results/sweep-status/${target_slug}.json" \ + --base-size "$base_close_processed_limit" \ + --max-size "$max_close_processed_limit" > "$adaptive_batch_env" + cat "$adaptive_batch_env" + close_processed_limit="$(awk -F= '$1 == "close_processed_limit" { print $2 }' "$adaptive_batch_env")" + adaptive_apply_scan_reason="$(awk -F= '$1 == "adaptive_apply_scan_reason" { print $2 }' "$adaptive_batch_env")" +} + +summarize_apply_candidate_quality() { + candidate_quality_summary="not evaluated" + candidate_quality_detail="" + if [ "$sync_comments_only" = "true" ]; then + return + fi + local quality_args=( + --target-repo "$TARGET_REPO" + --apply-kind "$apply_kind" + --apply-close-reasons "$apply_close_reasons" + --stale-min-age-days "$stale_min_age_days" + --min-age-days "$min_age_days" + --min-age-minutes "$min_age_minutes" + ) + if [ -n "$item_numbers" ]; then + quality_args+=(--item-numbers "$item_numbers") + else + quality_args+=(--batch-size "$close_processed_limit" --cursor-path "$apply_cursor_path") + fi + local candidate_quality_env=".artifacts/apply-candidate-quality.env" + pnpm run --silent workflow -- proposed-item-quality-summary "${quality_args[@]}" > "$candidate_quality_env" + cat "$candidate_quality_env" + candidate_quality_summary="$(awk -F= '$1 == "candidate_quality_summary" { print $2 }' "$candidate_quality_env")" + candidate_quality_detail=" Close candidate mix: $candidate_quality_summary." +} diff --git a/src/clawsweeper.ts b/src/clawsweeper.ts index 88bd6e3fab..48eac6a2b9 100644 --- a/src/clawsweeper.ts +++ b/src/clawsweeper.ts @@ -788,6 +788,7 @@ interface WorkflowStatusSummary { detail: string; runUrl: string | undefined; applyHealth: Record | undefined; + lastCloseApplyHealth: Record | undefined; plannedCount: number | undefined; plannedCapacity: number | undefined; plannedShards: number | undefined; @@ -1954,10 +1955,14 @@ function writeSweepStatus(options: { }): void { const profile = options.profile ?? targetProfile(); const updatedAt = new Date().toISOString(); + const previousStatus = readSweepStatusSummary(profile); const applyHealth = - options.applyHealth === undefined - ? readSweepStatusSummary(profile)?.applyHealth - : options.applyHealth; + options.applyHealth === undefined ? previousStatus?.applyHealth : options.applyHealth; + const previousCloseApplyHealth = + previousStatus?.lastCloseApplyHealth ?? + (previousStatus?.applyHealth?.mode === "close" ? previousStatus.applyHealth : undefined); + const lastCloseApplyHealth = + applyHealth && applyHealth.mode === "close" ? applyHealth : previousCloseApplyHealth; const payload = { schema_version: 1, slug: profile.slug, @@ -1980,6 +1985,7 @@ function writeSweepStatus(options: { bot_owned_proof_decisions_requested: options.botOwnedProofDecisionsRequested ?? null, bot_owned_proof_dispatches: options.botOwnedProofDispatches ?? null, apply_health: applyHealth ?? null, + last_close_apply_health: lastCloseApplyHealth ?? null, updated_at: updatedAt, }; const outputPath = sweepStatusPath(profile); @@ -8265,6 +8271,7 @@ function readSweepStatusSummary(profile = targetProfile()): WorkflowStatusSummar detail: stringOrUndefined(parsed.detail) ?? "No workflow status has been published yet.", runUrl: stringOrUndefined(parsed.run_url), applyHealth: recordOrUndefined(parsed.apply_health), + lastCloseApplyHealth: recordOrUndefined(parsed.last_close_apply_health), plannedCount: numberOrUndefined(parsed.planned_count), plannedCapacity: numberOrUndefined(parsed.planned_capacity), plannedShards: numberOrUndefined(parsed.planned_shards), @@ -8359,6 +8366,7 @@ function workflowStatusSummary(block: string): WorkflowStatusSummary { detail, runUrl, applyHealth: undefined, + lastCloseApplyHealth: undefined, plannedCount: numberOrUndefined(planMatch?.[1]), plannedShards: numberOrUndefined(planMatch?.[2]), plannedCapacity: numberOrUndefined(planMatch?.[3]), diff --git a/src/repair/workflow-utils.ts b/src/repair/workflow-utils.ts index c0dfd38bc5..b626c1b185 100644 --- a/src/repair/workflow-utils.ts +++ b/src/repair/workflow-utils.ts @@ -113,6 +113,25 @@ type ApplyCycleSummary = { scheduled_interval_minutes: number | null; label: string; }; + +type AdaptiveApplyBatchSizeOptions = { + statusPath: string; + baseSize: number; + maxSize: number; +}; + +type AdaptiveApplyBatchSize = { + closeProcessedLimit: number; + baseCloseProcessedLimit: number; + maxCloseProcessedLimit: number; + adaptive: boolean; + reason: string; + previousProcessed: number | null; + previousProcessedLimit: number | null; + previousClosed: number | null; + previousSkipped: number | null; +}; + const args = parseArgs(process.argv.slice(2)); function runCli(): void { @@ -181,6 +200,25 @@ function runCli(): void { )}\n`, ); break; + case "adaptive-apply-batch-size": { + const result = adaptiveApplyBatchSize({ + statusPath: requiredString("status-path"), + baseSize: numberArg("base-size", 300), + maxSize: numberArg("max-size", 900), + }); + printOutput({ + close_processed_limit: String(result.closeProcessedLimit), + base_close_processed_limit: String(result.baseCloseProcessedLimit), + max_close_processed_limit: String(result.maxCloseProcessedLimit), + adaptive_apply_scan: result.adaptive ? "true" : "false", + adaptive_apply_scan_reason: result.reason, + previous_apply_processed: optionalNumberOutput(result.previousProcessed), + previous_apply_processed_limit: optionalNumberOutput(result.previousProcessedLimit), + previous_apply_closed: optionalNumberOutput(result.previousClosed), + previous_apply_skipped: optionalNumberOutput(result.previousSkipped), + }); + break; + } case "count-command-actions": console.log( countCommandActions( @@ -334,6 +372,89 @@ function defaultCapacityReason( return "under capacity: due backlog below planned capacity"; } +export function adaptiveApplyBatchSize( + options: AdaptiveApplyBatchSizeOptions, +): AdaptiveApplyBatchSize { + const baseSize = positiveInteger(options.baseSize, "baseSize"); + const maxSize = Math.max(baseSize, positiveInteger(options.maxSize, "maxSize")); + const base: AdaptiveApplyBatchSize = { + closeProcessedLimit: baseSize, + baseCloseProcessedLimit: baseSize, + maxCloseProcessedLimit: maxSize, + adaptive: false, + reason: "base_window", + previousProcessed: null, + previousProcessedLimit: null, + previousClosed: null, + previousSkipped: null, + }; + const status = readJsonObjectIfPresent(options.statusPath); + if (!status) return base; + const health = closeApplyHealthFromStatus(status); + if (!health) return base; + const previousProcessed = nonNegativeIntegerOrNull(health.processed); + const previousProcessedLimit = nonNegativeIntegerOrNull(health.processed_limit); + const previousClosed = nonNegativeIntegerOrNull(health.closed); + const previousSkipped = nonNegativeIntegerOrNull(health.skipped); + const withPrevious = { + ...base, + previousProcessed, + previousProcessedLimit, + previousClosed, + previousSkipped, + }; + if (health.cursor_required !== true) return withPrevious; + if ( + previousProcessed === null || + previousProcessedLimit === null || + previousClosed === null || + previousSkipped === null + ) { + return withPrevious; + } + + const fullWindow = previousProcessedLimit > 0 && previousProcessed >= previousProcessedLimit; + const zeroClose = previousClosed === 0; + const skipHeavy = previousProcessed > 0 && previousSkipped / previousProcessed >= 0.8; + const attentionReasons = new Set(stringArray(health.attention_reasons)); + const unsafeAttention = [ + "cursor_required_but_missing_after_full_window", + "skipped_live_fetch_failed", + "skipped_runtime_budget", + ].some((reason) => attentionReasons.has(reason)); + if (!fullWindow || !zeroClose || !skipHeavy || unsafeAttention) return withPrevious; + + const grown = Math.min(maxSize, Math.max(baseSize, previousProcessedLimit * 2)); + if (grown <= baseSize) return withPrevious; + return { + ...withPrevious, + closeProcessedLimit: grown, + adaptive: true, + reason: "previous_full_zero_close_skip_window", + }; +} + +function closeApplyHealthFromStatus(status: LooseRecord): LooseRecord | null { + const current = recordOrNull(status.apply_health); + if (current?.mode === "close") return current; + const preserved = recordOrNull(status.last_close_apply_health); + return preserved?.mode === "close" ? preserved : null; +} + +function optionalNumberOutput(value: number | null): string { + return value === null ? "" : String(value); +} + +function positiveInteger(value: number, name: string): number { + if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`); + return value; +} + +function nonNegativeIntegerOrNull(value: JsonValue | undefined): number | null { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : null; +} + export function plannedItemNumberCsv(plan: LooseRecord): string { const candidates: JsonValue[] = Array.isArray(plan.candidates) ? plan.candidates : []; return candidates @@ -1570,6 +1691,19 @@ function readJsonObject(filePath: string): LooseRecord { return parsed; } +function readJsonObjectIfPresent(filePath: string): LooseRecord | null { + if (!filePath || !fs.existsSync(filePath)) return null; + try { + return readJsonObject(filePath); + } catch { + return null; + } +} + +function recordOrNull(value: JsonValue | undefined): LooseRecord | null { + return isJsonObject(value) ? value : null; +} + function readJsonArray(filePath: string): unknown[] { const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf8")); if (!Array.isArray(parsed)) throw new Error(`${filePath} must contain a JSON array`); @@ -1633,7 +1767,7 @@ function positiveNumber(value: string, fallback: number): number { return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } -function stringArray(value: JsonValue): string[] { +function stringArray(value: JsonValue | undefined): string[] { return Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === "string") : []; diff --git a/test/clawsweeper.test.ts b/test/clawsweeper.test.ts index c23b411bdf..8ff8af0afb 100644 --- a/test/clawsweeper.test.ts +++ b/test/clawsweeper.test.ts @@ -1996,6 +1996,9 @@ test("sweep status writer preserves non-apply health and clears stale apply upda /state\.startsWith\("Apply "\)\s+\?\s+null\s+:\s+readSweepStatusSummary\(profile\)\?\.applyHealth/, ); assert.match(source, /apply_health: applyHealth \?\? null/); + assert.match(source, /last_close_apply_health: lastCloseApplyHealth \?\? null/); + assert.match(source, /previousStatus\?\.lastCloseApplyHealth/); + assert.match(source, /previousStatus\?\.applyHealth\?\.mode === "close"/); }); test("review parser strips environment access caveats from risks", () => { diff --git a/test/repair/workflow-utils.test.ts b/test/repair/workflow-utils.test.ts index 105cf4ddae..e17ea91d98 100644 --- a/test/repair/workflow-utils.test.ts +++ b/test/repair/workflow-utils.test.ts @@ -8,6 +8,7 @@ import test from "node:test"; import { applyContinuationBlocker, applyCursorAdvanceCount, + adaptiveApplyBatchSize, artifactItemNumbers, automationLimit, commentSyncBatchOutput, @@ -750,6 +751,136 @@ test("workflow utilities expose review capacity telemetry from plans", () => { ); }); +test("workflow utilities expand automatic apply scan after a skip-heavy zero-close window", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "clawsweeper-workflow-")); + const statusPath = path.join(root, "results/sweep-status/openclaw-openclaw.json"); + write( + statusPath, + JSON.stringify({ + apply_health: { + mode: "close", + cursor_required: true, + processed: 300, + processed_limit: 300, + closed: 0, + skipped: 285, + attention_reasons: ["skipped_changed_since_review"], + }, + }), + ); + + const result = adaptiveApplyBatchSize({ statusPath, baseSize: 300, maxSize: 900 }); + + assert.equal(result.closeProcessedLimit, 600); + assert.equal(result.adaptive, true); + assert.equal(result.reason, "previous_full_zero_close_skip_window"); + assert.equal(result.previousProcessed, 300); + assert.equal(result.previousSkipped, 285); +}); + +test("workflow utilities use preserved close health after comment-sync status updates", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "clawsweeper-workflow-")); + const statusPath = path.join(root, "results/sweep-status/openclaw-openclaw.json"); + write( + statusPath, + JSON.stringify({ + apply_health: { + mode: "comment_sync", + cursor_required: true, + processed: 25, + processed_limit: 25, + closed: 0, + skipped: 0, + attention_reasons: [], + }, + last_close_apply_health: { + mode: "close", + cursor_required: true, + processed: 300, + processed_limit: 300, + closed: 0, + skipped: 300, + attention_reasons: ["skipped_changed_since_review"], + }, + }), + ); + + const result = adaptiveApplyBatchSize({ statusPath, baseSize: 300, maxSize: 900 }); + + assert.equal(result.closeProcessedLimit, 600); + assert.equal(result.adaptive, true); + assert.equal(result.previousProcessed, 300); +}); + +test("workflow utilities cap adaptive apply scan and reset on productive or unsafe windows", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "clawsweeper-workflow-")); + const statusPath = path.join(root, "results/sweep-status/openclaw-openclaw.json"); + const size = () => adaptiveApplyBatchSize({ statusPath, baseSize: 300, maxSize: 900 }); + + write( + statusPath, + JSON.stringify({ + apply_health: { + mode: "close", + cursor_required: true, + processed: 600, + processed_limit: 600, + closed: 0, + skipped: 600, + attention_reasons: ["skipped_protected_label"], + }, + }), + ); + assert.equal(size().closeProcessedLimit, 900); + + write( + statusPath, + JSON.stringify({ + apply_health: { + mode: "close", + cursor_required: true, + processed: 300, + processed_limit: 300, + closed: 1, + skipped: 299, + attention_reasons: [], + }, + }), + ); + assert.deepEqual( + { limit: size().closeProcessedLimit, reason: size().reason, adaptive: size().adaptive }, + { limit: 300, reason: "base_window", adaptive: false }, + ); + + write( + statusPath, + JSON.stringify({ + apply_health: { + mode: "close", + cursor_required: true, + processed: 300, + processed_limit: 300, + closed: 0, + skipped: 300, + attention_reasons: ["skipped_live_fetch_failed"], + }, + }), + ); + assert.deepEqual( + { limit: size().closeProcessedLimit, reason: size().reason, adaptive: size().adaptive }, + { limit: 300, reason: "base_window", adaptive: false }, + ); + + assert.equal( + adaptiveApplyBatchSize({ + statusPath: path.join(root, "missing.json"), + baseSize: 300, + maxSize: 900, + }).closeProcessedLimit, + 300, + ); +}); + test("workflow utilities select eligible proposed close records", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "clawsweeper-workflow-")); const oldDate = "2024-01-01T00:00:00Z"; diff --git a/test/sweep-workflow.test.ts b/test/sweep-workflow.test.ts index 000723e081..e0689fe098 100644 --- a/test/sweep-workflow.test.ts +++ b/test/sweep-workflow.test.ts @@ -122,6 +122,7 @@ test("apply workflow installs Codex only when proof-eligible apply work can run" test("apply workflow bounds checkpoints and requeues with a fresh token", () => { const workflow = readText(".github/workflows/sweep.yml"); + const applyHelper = readText("scripts/apply-workflow-helpers.sh"); const inputBlock = workflow.slice( workflow.indexOf(" workflow_dispatch:\n inputs:"), workflow.indexOf("\n schedule:"), @@ -135,6 +136,14 @@ test("apply workflow bounds checkpoints and requeues with a fresh token", () => applyJob.indexOf("- name: Continue apply sweep"), applyJob.indexOf("- name: Queue review backstops"), ); + const runMarker = " run: |\n"; + const runBodyStart = applyStep.indexOf(runMarker); + assert.notEqual(runBodyStart, -1); + const runBody = applyStep + .slice(runBodyStart + runMarker.length) + .split("\n") + .map((line) => (line.startsWith(" ") ? line.slice(10) : line)) + .join("\n"); assert.match(workflow, /format\('Apply default ClawSweeper closures for \{0\}'/); assert.match(workflow, /format\('Apply custom ClawSweeper closures for \{0\}'/); @@ -145,7 +154,17 @@ test("apply workflow bounds checkpoints and requeues with a fresh token", () => assert.match(inputBlock, /apply_limit:[\s\S]*default: "20"/); assert.match(inputBlock, /apply_checkpoint_size:[\s\S]*default: "20"/); assert.match(applyStep, /Capping apply checkpoint size at 20/); - assert.match(applyStep, /close_processed_limit=300/); + assert.match(applyStep, /base_close_processed_limit=300/); + assert.match(applyStep, /max_close_processed_limit=900/); + assert.match(applyStep, /close_processed_limit="\$base_close_processed_limit"/); + assert.match(applyStep, /source scripts\/apply-workflow-helpers\.sh/); + assert.match(applyStep, /select_adaptive_apply_batch/); + assert.match(applyHelper, /adaptive-apply-batch-size/); + assert.match(applyHelper, /--status-path "results\/sweep-status\/\$\{target_slug\}\.json"/); + assert.ok( + runBody.length < 20_000, + `apply run expression is ${runBody.length} characters; keep margin below GitHub's 21,000-character limit`, + ); assert.match(applyStep, /processed-limit "\$close_processed_limit"/); assert.match(applyStep, /comment_sync_processed_limit=1000/); assert.match(applyStep, /--processed-limit "\$comment_sync_processed_limit"/); @@ -153,14 +172,14 @@ test("apply workflow bounds checkpoints and requeues with a fresh token", () => assert.ok(applyFlagInit > applyStep.indexOf('item_numbers="${{')); assert.ok(applyFlagInit < applyStep.indexOf("auto_selected_apply_batch=true")); assert.match(applyStep, /apply_cursor_path="results\/apply-cursors\/\$\{target_slug\}\.json"/); - assert.match(applyStep, /write_apply_health\(\)/); + assert.match(applyHelper, /write_apply_health\(\)/); assert.match(applyStep, /proposed-item-count/); assert.match(applyStep, /apply-cursor-advance-count/); - assert.match(applyStep, /--candidate-count "\$health_candidate_count"/); - assert.match(applyStep, /--cursor-advance-count "\$health_cursor_advance_count"/); - assert.match(applyStep, /--scheduled-interval-minutes "\$health_scheduled_interval_minutes"/); - assert.match(applyStep, /pnpm run --silent workflow -- summarize-apply-report/); - assert.match(applyStep, /health_cursor_path="\$\{5:-\}"/); + assert.match(applyHelper, /--candidate-count "\$health_candidate_count"/); + assert.match(applyHelper, /--cursor-advance-count "\$health_cursor_advance_count"/); + assert.match(applyHelper, /--scheduled-interval-minutes "\$health_scheduled_interval_minutes"/); + assert.match(applyHelper, /pnpm run --silent workflow -- summarize-apply-report/); + assert.match(applyHelper, /health_cursor_path="\$\{5:-\}"/); assert.match(applyStep, /comment_sync_health_cursor_path="\$cursor_path"/); assert.match(applyStep, /comment_sync_health_cursor_required="true"/); assert.match(applyStep, /comment_sync_health_processed_limit="\$sync_batch_size"/); @@ -168,21 +187,32 @@ test("apply workflow bounds checkpoints and requeues with a fresh token", () => assert.match(applyStep, /--apply-health-file "\.artifacts\/apply-health-\$checkpoint\.json"/); assert.match(applyStep, /--apply-health-file "\.artifacts\/apply-health-final\.json"/); assert.match(applyStep, /--state "Apply idle"/); - assert.match(applyStep, /proposed-item-quality-summary/); - assert.match(applyStep, /candidate_quality_summary="\$\(awk -F=/); + assert.match(applyHelper, /proposed-item-quality-summary/); + assert.match(applyHelper, /candidate_quality_summary="\$\(awk -F=/); assert.match( - applyStep, + applyHelper, /candidate_quality_detail=" Close candidate mix: \$candidate_quality_summary\."/, ); assert.match(applyStep, /awaiting apply\.\$candidate_quality_detail Scheduled apply/); - assert.match(applyStep, /\$apply_close_reasons\.\$candidate_quality_detail Existing Codex/); + assert.match( + applyStep, + /\$apply_close_reasons\.\$candidate_quality_detail Scan window: \$close_processed_limit/, + ); const applyReconcileIndex = applyStep.indexOf('pnpm run reconcile -- "${reconcile_args[@]}"'); - const qualitySummaryIndex = applyStep.indexOf("proposed-item-quality-summary"); + const qualitySummaryIndex = applyStep.indexOf("summarize_apply_candidate_quality"); const proposedNumbersIndex = applyStep.indexOf("proposed-item-numbers"); assert.notEqual(applyReconcileIndex, -1); assert.ok(qualitySummaryIndex > applyReconcileIndex); assert.ok(proposedNumbersIndex > qualitySummaryIndex); assert.match(applyStep, /--batch-size "\$close_processed_limit"/); + assert.match( + applyStep, + /Scan window: \$close_processed_limit records \(\$adaptive_apply_scan_reason\)/, + ); + assert.match( + applyStep, + /Auto-selected \$proposed_count proposed close candidate\(s\) from \$close_processed_limit-record apply cursor window/, + ); assert.match(applyStep, /--cursor-path "\$apply_cursor_path"/); assert.match(applyStep, /write-apply-cursor/); assert.match(applyStep, /--item-numbers "\$item_numbers"/); From fb2d036e19bea416613b791c3cbdc42e5ec1a71b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 4 Jul 2026 07:09:23 -0400 Subject: [PATCH 2/2] fix: keep the apply workflow under GitHub's expression limit Move reusable apply helpers out of the inline run expression and add a regression ceiling below GitHub's 21,000-character validation limit. --- .github/workflows/sweep.yml | 88 ++++++------------------------------- 1 file changed, 13 insertions(+), 75 deletions(-) diff --git a/.github/workflows/sweep.yml b/.github/workflows/sweep.yml index db5f1a0165..37d584ca1d 100644 --- a/.github/workflows/sweep.yml +++ b/.github/workflows/sweep.yml @@ -2310,6 +2310,7 @@ jobs: run: | set -euo pipefail test -n "$GH_TOKEN" + source scripts/apply-workflow-helpers.sh limit="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_limit || '20' }}" min_age_days="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_min_age_days || '0' }}" min_age_minutes="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_min_age_minutes || '' }}" @@ -2320,7 +2321,12 @@ jobs: progress_every="10" checkpoint_size="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_checkpoint_size || '20' }}" comment_sync_processed_limit=1000 - close_processed_limit=300 + base_close_processed_limit=300 + # Consumed by the sourced workflow helpers. + # shellcheck disable=SC2034 + max_close_processed_limit=900 + close_processed_limit="$base_close_processed_limit" + adaptive_apply_scan_reason="base_window" sync_batch_size="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_limit || '25' }}" item_numbers="${{ github.event_name == 'repository_dispatch' && github.event.client_payload.item_number || github.event.inputs.apply_item_numbers || '' }}" sync_open_pr_batch="${{ github.event_name == 'schedule' && github.event.schedule == '6,21,36,51 * * * *' && 'true' || 'false' }}" @@ -2368,84 +2374,14 @@ jobs: echo "Capping apply checkpoint size at 20 to keep each GitHub App token within its lifetime." checkpoint_size=20 fi - candidate_quality_summary="not evaluated" - candidate_quality_detail="" + select_adaptive_apply_batch mkdir -p .artifacts/apply-reports - publish_changes() { - message="$1" - shift - publish_args=(--message "$message" --rebase-strategy apply-records) - for path in "$@"; do - publish_args+=(--path "$path") - done - pnpm run repair:publish-main -- "${publish_args[@]}" - } - publish_status() { - message="$1" - if ! publish_changes "$message" results/sweep-status; then - echo "Best-effort status update failed: $message" - git restore results/sweep-status || true - fi - } - write_apply_health() { - report_path="$1" - output_path="$2" - health_mode="$3" - health_processed_limit="$4" - health_cursor_path="${5:-}" - health_cursor_required="${6:-false}" - health_candidate_count="${7:-}" - health_scheduled_interval_minutes="${8:-}" - health_cursor_advance_count="${9:-}" - health_args=( - --target-repo "$TARGET_REPO" - --report "$report_path" - --mode "$health_mode" - --processed-limit "$health_processed_limit" - --close-limit "$limit" - ) - if [ -n "$health_cursor_path" ]; then - health_args+=(--cursor-path "$health_cursor_path") - fi - if [ "$health_cursor_required" = "true" ]; then - health_args+=(--cursor-required true) - fi - if [ -n "$health_candidate_count" ]; then - health_args+=(--candidate-count "$health_candidate_count") - fi - if [ -n "$health_scheduled_interval_minutes" ]; then - health_args+=(--scheduled-interval-minutes "$health_scheduled_interval_minutes") - fi - if [ -n "$health_cursor_advance_count" ]; then - health_args+=(--cursor-advance-count "$health_cursor_advance_count") - fi - pnpm run --silent workflow -- summarize-apply-report "${health_args[@]}" > "$output_path" - } reconcile_args=(--target-repo "$TARGET_REPO" --skip-closed-at) if [ -n "$item_numbers" ]; then reconcile_args+=(--item-numbers "$item_numbers") fi pnpm run reconcile -- "${reconcile_args[@]}" - if [ "$sync_comments_only" != "true" ]; then - quality_args=( - --target-repo "$TARGET_REPO" - --apply-kind "$apply_kind" - --apply-close-reasons "$apply_close_reasons" - --stale-min-age-days "$stale_min_age_days" - --min-age-days "$min_age_days" - --min-age-minutes "$min_age_minutes" - ) - if [ -n "$item_numbers" ]; then - quality_args+=(--item-numbers "$item_numbers") - else - quality_args+=(--batch-size "$close_processed_limit" --cursor-path "$apply_cursor_path") - fi - candidate_quality_env=".artifacts/apply-candidate-quality.env" - pnpm run --silent workflow -- proposed-item-quality-summary "${quality_args[@]}" > "$candidate_quality_env" - cat "$candidate_quality_env" - candidate_quality_summary="$(awk -F= '$1 == "candidate_quality_summary" { print $2 }' "$candidate_quality_env")" - candidate_quality_detail=" Close candidate mix: $candidate_quality_summary." - fi + summarize_apply_candidate_quality if [ "$sync_open_pr_batch" = "true" ] && [ -z "$item_numbers" ]; then batch_env=".artifacts/comment-sync-batch.env" pnpm run --silent workflow -- comment-sync-batch \ @@ -2509,7 +2445,7 @@ jobs: if [ "$proposed_count" -lt "$limit" ]; then limit="$proposed_count" fi - echo "Auto-selected $proposed_count proposed close candidate(s) from apply cursor window: $item_numbers; apply-ready count: ${apply_ready_count:-unknown}" + echo "Auto-selected $proposed_count proposed close candidate(s) from $close_processed_limit-record apply cursor window ($adaptive_apply_scan_reason): $item_numbers; apply-ready count: ${apply_ready_count:-unknown}" fi fi item_numbers_arg=() @@ -2545,7 +2481,7 @@ jobs: pnpm run status -- \ --target-repo "$TARGET_REPO" \ --state "Apply in progress" \ - --detail "Starting apply/comment-sync run for up to $limit fresh $apply_kind closes. Close reasons: $apply_close_reasons.$candidate_quality_detail Existing Codex automated review comments are updated in place when closing or when comment-only sync is stale by ${comment_sync_min_age_days} day(s); checkpoints commit every $checkpoint_size fresh closes; close delay is ${close_delay_ms}ms; sync-comments-only=$sync_comments_only; item numbers=${item_numbers:-all}." \ + --detail "Starting apply/comment-sync run for up to $limit fresh $apply_kind closes. Close reasons: $apply_close_reasons.$candidate_quality_detail Scan window: $close_processed_limit records ($adaptive_apply_scan_reason). Existing Codex automated review comments are updated in place when closing or when comment-only sync is stale by ${comment_sync_min_age_days} day(s); checkpoints commit every $checkpoint_size fresh closes; close delay is ${close_delay_ms}ms; sync-comments-only=$sync_comments_only; item numbers=${item_numbers:-all}." \ --run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" if ! publish_changes "chore: mark sweep apply in progress" records results/sweep-status; then echo "Best-effort apply start status update failed" @@ -2738,6 +2674,8 @@ jobs: echo "APPLY_SYNC_OPEN_PR_BATCH=$sync_open_pr_batch" echo "APPLY_AUTO_SELECTED_BATCH=$auto_selected_apply_batch" echo "APPLY_COMMENT_SYNC_MIN_AGE_DAYS=$comment_sync_min_age_days" + echo "APPLY_CLOSE_PROCESSED_LIMIT=$close_processed_limit" + echo "APPLY_ADAPTIVE_SCAN_REASON=$adaptive_apply_scan_reason" echo "APPLY_CANDIDATE_QUALITY_SUMMARY=$candidate_quality_summary" } >> "$GITHUB_ENV"