feat: adapt apply scan window

.github/workflows/sweep.yml

@@ -2310,6 +2310,7 @@ jobs:
         run: |
           set -euo pipefail
           test -n "$GH_TOKEN"
+          source scripts/apply-workflow-helpers.sh
           limit="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_limit || '20' }}"
           min_age_days="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_min_age_days || '0' }}"
           min_age_minutes="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_min_age_minutes || '' }}"
@@ -2320,7 +2321,12 @@ jobs:
           progress_every="10"
           checkpoint_size="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_checkpoint_size || '20' }}"
           comment_sync_processed_limit=1000
-          close_processed_limit=300
+          base_close_processed_limit=300
+          # Consumed by the sourced workflow helpers.
+          # shellcheck disable=SC2034
+          max_close_processed_limit=900
+          close_processed_limit="$base_close_processed_limit"
+          adaptive_apply_scan_reason="base_window"
           sync_batch_size="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.apply_limit || '25' }}"
           item_numbers="${{ github.event_name == 'repository_dispatch' && github.event.client_payload.item_number || github.event.inputs.apply_item_numbers || '' }}"
           sync_open_pr_batch="${{ github.event_name == 'schedule' && github.event.schedule == '6,21,36,51 * * * *' && 'true' || 'false' }}"
@@ -2368,84 +2374,14 @@ jobs:
             echo "Capping apply checkpoint size at 20 to keep each GitHub App token within its lifetime."
             checkpoint_size=20
           fi
-          candidate_quality_summary="not evaluated"
-          candidate_quality_detail=""
+          select_adaptive_apply_batch
           mkdir -p .artifacts/apply-reports
-          publish_changes() {
-            message="$1"
-            shift
-            publish_args=(--message "$message" --rebase-strategy apply-records)
-            for path in "$@"; do
-              publish_args+=(--path "$path")
-            done
-            pnpm run repair:publish-main -- "${publish_args[@]}"
-          }
-          publish_status() {
-            message="$1"
-            if ! publish_changes "$message" results/sweep-status; then
-              echo "Best-effort status update failed: $message"
-              git restore results/sweep-status || true
-            fi
-          }
-          write_apply_health() {
-            report_path="$1"
-            output_path="$2"
-            health_mode="$3"
-            health_processed_limit="$4"
-            health_cursor_path="${5:-}"
-            health_cursor_required="${6:-false}"
-            health_candidate_count="${7:-}"
-            health_scheduled_interval_minutes="${8:-}"
-            health_cursor_advance_count="${9:-}"
-            health_args=(
-              --target-repo "$TARGET_REPO"
-              --report "$report_path"
-              --mode "$health_mode"
-              --processed-limit "$health_processed_limit"
-              --close-limit "$limit"
-            )
-            if [ -n "$health_cursor_path" ]; then
-              health_args+=(--cursor-path "$health_cursor_path")
-            fi
-            if [ "$health_cursor_required" = "true" ]; then
-              health_args+=(--cursor-required true)
-            fi
-            if [ -n "$health_candidate_count" ]; then
-              health_args+=(--candidate-count "$health_candidate_count")
-            fi
-            if [ -n "$health_scheduled_interval_minutes" ]; then
-              health_args+=(--scheduled-interval-minutes "$health_scheduled_interval_minutes")
-            fi
-            if [ -n "$health_cursor_advance_count" ]; then
-              health_args+=(--cursor-advance-count "$health_cursor_advance_count")
-            fi
-            pnpm run --silent workflow -- summarize-apply-report "${health_args[@]}" > "$output_path"
-          }
           reconcile_args=(--target-repo "$TARGET_REPO" --skip-closed-at)
           if [ -n "$item_numbers" ]; then
             reconcile_args+=(--item-numbers "$item_numbers")
           fi
           pnpm run reconcile -- "${reconcile_args[@]}"
-          if [ "$sync_comments_only" != "true" ]; then
-            quality_args=(
-              --target-repo "$TARGET_REPO"
-              --apply-kind "$apply_kind"
-              --apply-close-reasons "$apply_close_reasons"
-              --stale-min-age-days "$stale_min_age_days"
-              --min-age-days "$min_age_days"
-              --min-age-minutes "$min_age_minutes"
-            )
-            if [ -n "$item_numbers" ]; then
-              quality_args+=(--item-numbers "$item_numbers")
-            else
-              quality_args+=(--batch-size "$close_processed_limit" --cursor-path "$apply_cursor_path")
-            fi
-            candidate_quality_env=".artifacts/apply-candidate-quality.env"
-            pnpm run --silent workflow -- proposed-item-quality-summary "${quality_args[@]}" > "$candidate_quality_env"
-            cat "$candidate_quality_env"
-            candidate_quality_summary="$(awk -F= '$1 == "candidate_quality_summary" { print $2 }' "$candidate_quality_env")"
-            candidate_quality_detail=" Close candidate mix: $candidate_quality_summary."
-          fi
+          summarize_apply_candidate_quality
           if [ "$sync_open_pr_batch" = "true" ] && [ -z "$item_numbers" ]; then
             batch_env=".artifacts/comment-sync-batch.env"
             pnpm run --silent workflow -- comment-sync-batch \
@@ -2509,7 +2445,7 @@ jobs:
               if [ "$proposed_count" -lt "$limit" ]; then
                 limit="$proposed_count"
               fi
-              echo "Auto-selected $proposed_count proposed close candidate(s) from apply cursor window: $item_numbers; apply-ready count: ${apply_ready_count:-unknown}"
+              echo "Auto-selected $proposed_count proposed close candidate(s) from $close_processed_limit-record apply cursor window ($adaptive_apply_scan_reason): $item_numbers; apply-ready count: ${apply_ready_count:-unknown}"
             fi
           fi
           item_numbers_arg=()
@@ -2545,7 +2481,7 @@ jobs:
           pnpm run status -- \
             --target-repo "$TARGET_REPO" \
             --state "Apply in progress" \
-            --detail "Starting apply/comment-sync run for up to $limit fresh $apply_kind closes. Close reasons: $apply_close_reasons.$candidate_quality_detail Existing Codex automated review comments are updated in place when closing or when comment-only sync is stale by ${comment_sync_min_age_days} day(s); checkpoints commit every $checkpoint_size fresh closes; close delay is ${close_delay_ms}ms; sync-comments-only=$sync_comments_only; item numbers=${item_numbers:-all}." \
+            --detail "Starting apply/comment-sync run for up to $limit fresh $apply_kind closes. Close reasons: $apply_close_reasons.$candidate_quality_detail Scan window: $close_processed_limit records ($adaptive_apply_scan_reason). Existing Codex automated review comments are updated in place when closing or when comment-only sync is stale by ${comment_sync_min_age_days} day(s); checkpoints commit every $checkpoint_size fresh closes; close delay is ${close_delay_ms}ms; sync-comments-only=$sync_comments_only; item numbers=${item_numbers:-all}." \
             --run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           if ! publish_changes "chore: mark sweep apply in progress" records results/sweep-status; then
             echo "Best-effort apply start status update failed"
@@ -2738,6 +2674,8 @@ jobs:
             echo "APPLY_SYNC_OPEN_PR_BATCH=$sync_open_pr_batch"
             echo "APPLY_AUTO_SELECTED_BATCH=$auto_selected_apply_batch"
             echo "APPLY_COMMENT_SYNC_MIN_AGE_DAYS=$comment_sync_min_age_days"
+            echo "APPLY_CLOSE_PROCESSED_LIMIT=$close_processed_limit"
+            echo "APPLY_ADAPTIVE_SCAN_REASON=$adaptive_apply_scan_reason"
             echo "APPLY_CANDIDATE_QUALITY_SUMMARY=$candidate_quality_summary"
           } >> "$GITHUB_ENV"
 

CHANGELOG.md

@@ -18,6 +18,7 @@ checkpoint, and status-only commits are intentionally omitted.
 
 ### Changed
 
+- Expanded untargeted close-apply scans from 300 toward a capped 900 records after skip-heavy zero-close windows without changing close or worker limits. Thanks @brokemac79.
 - Made ClawHub diversion comments a practical self-serve handoff with package-shape, manifest, configuration, documentation, usage, and smoke-proof guidance. Thanks @brokemac79.
 - Reduced duplicate GitHub API reads in each live-dashboard status snapshot and batched recent automerge hydration into one GraphQL request with a REST fallback. Thanks @brokemac79.
 - Raised the apply-existing close limit and checkpoint size from 5 to 20 fresh closes per run so continuation chains drain the proposal queue faster while each GitHub App token stays within its lifetime.
@@ -26,6 +27,7 @@ checkpoint, and status-only commits are intentionally omitted.
 
 ### Fixed
 
+- Split apply workflow helpers out of the oversized inline expression so GitHub can validate and start sweep runs again.
 - Bounded apply-existing checkpoints to five fresh closes, renewed the GitHub
   App token between continuation runs, and stopped zero-progress scans from
   chaining indefinitely.

docs/scheduler.md

@@ -459,6 +459,15 @@ continuation with a fresh GitHub App token after any checkpoint that closes at
 least one item. A saturated scan that closes nothing stops without chaining so
 the same records cannot create an unbounded runner loop.
 
+Untargeted cursor-based close apply starts with a 300-record scan window. If
+the previous cursor window was a full close-mode scan, closed nothing, skipped
+at least 80% of processed records, and did not hit a live-fetch, runtime-budget,
+or missing-cursor failure, the next automatic window expands to inspect more
+records, capped at 900. This changes only the deterministic scan window:
+`apply_limit`, checkpoint size, close gates, live-state checks, and maintainer
+policy gates stay unchanged. The workflow logs and sweep status detail include
+the selected scan window and reason.
+
 Before a close-mode apply run starts, the workflow summarizes the selected close
 candidate mix by quality bucket in the status detail. Buckets such as
 implemented-on-main, duplicate/superseded, needs PR close proof,

scripts/apply-workflow-helpers.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+# Shared by the apply workflow step. The caller supplies the current apply
+# settings as shell variables before sourcing this file.
+# shellcheck disable=SC2034,SC2154
+
+publish_changes() {
+  local message="$1"
+  shift
+  local publish_args=(--message "$message" --rebase-strategy apply-records)
+  local path
+  for path in "$@"; do
+    publish_args+=(--path "$path")
+  done
+  pnpm run repair:publish-main -- "${publish_args[@]}"
+}
+
+publish_status() {
+  local message="$1"
+  if ! publish_changes "$message" results/sweep-status; then
+    echo "Best-effort status update failed: $message"
+    git restore results/sweep-status || true
+  fi
+}
+
+write_apply_health() {
+  local report_path="$1"
+  local output_path="$2"
+  local health_mode="$3"
+  local health_processed_limit="$4"
+  local health_cursor_path="${5:-}"
+  local health_cursor_required="${6:-false}"
+  local health_candidate_count="${7:-}"
+  local health_scheduled_interval_minutes="${8:-}"
+  local health_cursor_advance_count="${9:-}"
+  local health_args=(
+    --target-repo "$TARGET_REPO"
+    --report "$report_path"
+    --mode "$health_mode"
+    --processed-limit "$health_processed_limit"
+    --close-limit "$limit"
+  )
+  if [ -n "$health_cursor_path" ]; then
+    health_args+=(--cursor-path "$health_cursor_path")
+  fi
+  if [ "$health_cursor_required" = "true" ]; then
+    health_args+=(--cursor-required true)
+  fi
+  if [ -n "$health_candidate_count" ]; then
+    health_args+=(--candidate-count "$health_candidate_count")
+  fi
+  if [ -n "$health_scheduled_interval_minutes" ]; then
+    health_args+=(--scheduled-interval-minutes "$health_scheduled_interval_minutes")
+  fi
+  if [ -n "$health_cursor_advance_count" ]; then
+    health_args+=(--cursor-advance-count "$health_cursor_advance_count")
+  fi
+  pnpm run --silent workflow -- summarize-apply-report "${health_args[@]}" > "$output_path"
+}
+
+select_adaptive_apply_batch() {
+  if [ "$sync_comments_only" = "true" ] || [ -n "$item_numbers" ]; then
+    return
+  fi
+  mkdir -p .artifacts
+  local adaptive_batch_env=".artifacts/apply-adaptive-batch.env"
+  pnpm run --silent workflow -- adaptive-apply-batch-size \
+    --status-path "results/sweep-status/${target_slug}.json" \
+    --base-size "$base_close_processed_limit" \
+    --max-size "$max_close_processed_limit" > "$adaptive_batch_env"
+  cat "$adaptive_batch_env"
+  close_processed_limit="$(awk -F= '$1 == "close_processed_limit" { print $2 }' "$adaptive_batch_env")"
+  adaptive_apply_scan_reason="$(awk -F= '$1 == "adaptive_apply_scan_reason" { print $2 }' "$adaptive_batch_env")"
+}
+
+summarize_apply_candidate_quality() {
+  candidate_quality_summary="not evaluated"
+  candidate_quality_detail=""
+  if [ "$sync_comments_only" = "true" ]; then
+    return
+  fi
+  local quality_args=(
+    --target-repo "$TARGET_REPO"
+    --apply-kind "$apply_kind"
+    --apply-close-reasons "$apply_close_reasons"
+    --stale-min-age-days "$stale_min_age_days"
+    --min-age-days "$min_age_days"
+    --min-age-minutes "$min_age_minutes"
+  )
+  if [ -n "$item_numbers" ]; then
+    quality_args+=(--item-numbers "$item_numbers")
+  else
+    quality_args+=(--batch-size "$close_processed_limit" --cursor-path "$apply_cursor_path")
+  fi
+  local candidate_quality_env=".artifacts/apply-candidate-quality.env"
+  pnpm run --silent workflow -- proposed-item-quality-summary "${quality_args[@]}" > "$candidate_quality_env"
+  cat "$candidate_quality_env"
+  candidate_quality_summary="$(awk -F= '$1 == "candidate_quality_summary" { print $2 }' "$candidate_quality_env")"
+  candidate_quality_detail=" Close candidate mix: $candidate_quality_summary."
+}

src/clawsweeper.ts

@@ -788,6 +788,7 @@ interface WorkflowStatusSummary {
   detail: string;
   runUrl: string | undefined;
   applyHealth: Record<string, unknown> | undefined;
+  lastCloseApplyHealth: Record<string, unknown> | undefined;
   plannedCount: number | undefined;
   plannedCapacity: number | undefined;
   plannedShards: number | undefined;
@@ -1954,10 +1955,14 @@ function writeSweepStatus(options: {
 }): void {
   const profile = options.profile ?? targetProfile();
   const updatedAt = new Date().toISOString();
+  const previousStatus = readSweepStatusSummary(profile);
   const applyHealth =
-    options.applyHealth === undefined
-      ? readSweepStatusSummary(profile)?.applyHealth
-      : options.applyHealth;
+    options.applyHealth === undefined ? previousStatus?.applyHealth : options.applyHealth;
+  const previousCloseApplyHealth =
+    previousStatus?.lastCloseApplyHealth ??
+    (previousStatus?.applyHealth?.mode === "close" ? previousStatus.applyHealth : undefined);
+  const lastCloseApplyHealth =
+    applyHealth && applyHealth.mode === "close" ? applyHealth : previousCloseApplyHealth;
   const payload = {
     schema_version: 1,
     slug: profile.slug,
@@ -1980,6 +1985,7 @@ function writeSweepStatus(options: {
     bot_owned_proof_decisions_requested: options.botOwnedProofDecisionsRequested ?? null,
     bot_owned_proof_dispatches: options.botOwnedProofDispatches ?? null,
     apply_health: applyHealth ?? null,
+    last_close_apply_health: lastCloseApplyHealth ?? null,
     updated_at: updatedAt,
   };
   const outputPath = sweepStatusPath(profile);
@@ -8265,6 +8271,7 @@ function readSweepStatusSummary(profile = targetProfile()): WorkflowStatusSummar
       detail: stringOrUndefined(parsed.detail) ?? "No workflow status has been published yet.",
       runUrl: stringOrUndefined(parsed.run_url),
       applyHealth: recordOrUndefined(parsed.apply_health),
+      lastCloseApplyHealth: recordOrUndefined(parsed.last_close_apply_health),
       plannedCount: numberOrUndefined(parsed.planned_count),
       plannedCapacity: numberOrUndefined(parsed.planned_capacity),
       plannedShards: numberOrUndefined(parsed.planned_shards),
@@ -8359,6 +8366,7 @@ function workflowStatusSummary(block: string): WorkflowStatusSummary {
     detail,
     runUrl,
     applyHealth: undefined,
+    lastCloseApplyHealth: undefined,
     plannedCount: numberOrUndefined(planMatch?.[1]),
     plannedShards: numberOrUndefined(planMatch?.[2]),
     plannedCapacity: numberOrUndefined(planMatch?.[3]),