From c98cae4667249bcafc07eb1d78ef4a742ea12a06 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:31:28 -0500 Subject: [PATCH 01/21] Prototype priority-aware CI scheduling --- .../workflows/benchmark-multinode-tmpl.yml | 24 +- .github/workflows/benchmark-tmpl.yml | 24 +- .github/workflows/collectivex-sweep.yml | 18 +- .github/workflows/e2e-tests.yml | 19 + .github/workflows/profile.yml | 19 +- .github/workflows/run-sweep.yml | 20 + .github/workflows/speedbench-al.yml | 19 +- configs/ci-priority.yaml | 51 +++ utils/ci_priority.py | 180 +++++++++ utils/ci_priority_controller.py | 344 ++++++++++++++++++ utils/test_ci_priority.py | 99 +++++ utils/test_ci_priority_controller.py | 151 ++++++++ 12 files changed, 959 insertions(+), 9 deletions(-) create mode 100644 configs/ci-priority.yaml create mode 100755 utils/ci_priority.py create mode 100755 utils/ci_priority_controller.py create mode 100644 utils/test_ci_priority.py create mode 100644 utils/test_ci_priority_controller.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index b2219144f1..0c45621417 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -6,6 +6,16 @@ on: runner: required: true type: string + priority: + description: "Lower-is-sooner CI priority score" + required: false + type: string + default: '5.000' + queue-token: + description: "One-shot queue token generated for this job" + required: false + type: string + default: 'legacy' image: required: true type: string @@ -231,10 +241,20 @@ permissions: jobs: benchmark: - runs-on: ${{ inputs.runner }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + format( + '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + inputs.runner, + inputs.priority, + inputs.queue-token + ) || + format('["{0}"]', inputs.runner) + ) }} timeout-minutes: 480 name: >- - ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} + p${{ inputs.priority }} | ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} ${{ inputs.prefill-num-worker }}P (TP${{ inputs.prefill-tp }}${{ inputs.prefill-pp != '' && inputs.prefill-pp != '1' && format('/PP{0}', inputs.prefill-pp) || '' }}${{ inputs.prefill-dcp-size != '' && inputs.prefill-dcp-size != '1' && format('/DCP{0}', inputs.prefill-dcp-size) || '' }}${{ inputs.prefill-pcp-size != '' && inputs.prefill-pcp-size != '1' && format('/PCP{0}', inputs.prefill-pcp-size) || '' }}${{ inputs.prefill-ep != '' && inputs.prefill-ep != '1' && format('/EP{0}', inputs.prefill-ep) || '' }}${{ inputs.prefill-dp-attn == 'true' && '/DPA' || '' }}) x ${{ inputs.decode-num-worker }}D (TP${{ inputs.decode-tp }}${{ inputs.decode-pp != '' && inputs.decode-pp != '1' && format('/PP{0}', inputs.decode-pp) || '' }}${{ inputs.decode-dcp-size != '' && inputs.decode-dcp-size != '1' && format('/DCP{0}', inputs.decode-dcp-size) || '' }}${{ inputs.decode-pcp-size != '' && inputs.decode-pcp-size != '1' && format('/PCP{0}', inputs.decode-pcp-size) || '' }}${{ inputs.decode-ep != '' && inputs.decode-ep != '1' && format('/EP{0}', inputs.decode-ep) || '' }}${{ inputs.decode-dp-attn == 'true' && '/DPA' || '' }}) ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }} diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 241dafc4a4..6d854989dc 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -5,6 +5,16 @@ on: runner: required: true type: string + priority: + description: "Lower-is-sooner CI priority score" + required: false + type: string + default: '5.000' + queue-token: + description: "One-shot queue token generated for this job" + required: false + type: string + default: 'legacy' image: required: true type: string @@ -161,10 +171,20 @@ permissions: jobs: benchmark: - runs-on: ${{ inputs.runner }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + format( + '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + inputs.runner, + inputs.priority, + inputs.queue-token + ) || + format('["{0}"]', inputs.runner) + ) }} timeout-minutes: 500 name: >- - ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} + p${{ inputs.priority }} | ${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }} TP${{ inputs.tp }}${{ inputs.pp != '' && inputs.pp != '1' && format('/PP{0}', inputs.pp) || '' }}${{ inputs.dcp-size != '' && inputs.dcp-size != '1' && format('/DCP{0}', inputs.dcp-size) || '' }}${{ inputs.pcp-size != '' && inputs.pcp-size != '1' && format('/PCP{0}', inputs.pcp-size) || '' }}${{ inputs.ep != '' && inputs.ep != '1' && format('/EP{0}', inputs.ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }} ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }} ${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }} diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml index b56bc285aa..a4b300cb46 100644 --- a/.github/workflows/collectivex-sweep.yml +++ b/.github/workflows/collectivex-sweep.yml @@ -32,6 +32,10 @@ on: description: Max cases per shard cell (chunk larger shards) type: string default: '14' + priority: + description: Lower-is-sooner CI priority score + type: string + default: '5.000' concurrency: group: cx-sweep-${{ github.ref }}-${{ inputs.backend }}-${{ inputs.deepep_v2 }}-${{ inputs.only_sku }} @@ -76,7 +80,19 @@ jobs: max-parallel: 10 # don't saturate the ~20-runner fleet; cells queue as slots free matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} # h200 label spans two clusters; pin to the validated dgxc pool (mirrors collectivex-experimental). - runs-on: ${{ matrix.sku == 'h200' && 'h200-dgxc' || matrix.sku }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + format( + '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}-{3}"]', + matrix.sku == 'h200' && 'h200-dgxc' || matrix.sku, + inputs.priority, + github.run_id, + matrix.id + ) || + format('["{0}"]', matrix.sku == 'h200' && 'h200-dgxc' || matrix.sku) + ) }} + name: p${{ inputs.priority }} | ${{ matrix.sku }} ${{ matrix.backend }} shard ${{ matrix.id }} timeout-minutes: 350 env: CX_BENCH: ${{ matrix.backend }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 51c68e1d85..06d9154ddb 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -65,10 +65,17 @@ jobs: ref: ${{ github.sha }} - id: get-jobs + env: + PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} run: | pip install pydantic CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py \ ${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}) + CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ + "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ + --event-name "${{ github.event_name }}" \ + --queue-namespace "${{ github.run_id }}" \ + --labels-json "$PR_LABELS") AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x]))") MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))") SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))") @@ -97,6 +104,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -146,6 +155,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -195,6 +206,8 @@ jobs: with: exp-name: ${{ matrix.config.exp-name }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -239,6 +252,8 @@ jobs: osl: '0' max-model-len: '0' runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -292,6 +307,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -327,6 +344,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index 10eb631d03..c352f42f58 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -56,6 +56,11 @@ jobs: pip install pydantic CLI_ARGS="test-config --config-files ${{ inputs.config-file }} --config-keys ${{ inputs.config-key }} --conc ${{ inputs.conc }}" CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py $CLI_ARGS) + CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ + "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ + --event-name "${{ github.event_name }}" \ + --queue-namespace "${{ github.run_id }}" \ + --labels-json '${{ inputs.moe-debug && '["ci-patchwork"]' || '[]' }}') echo "raw=$CONFIG_JSON" >> $GITHUB_OUTPUT - id: filter @@ -99,9 +104,19 @@ jobs: matrix: config: ${{ fromJson(needs.get-jobs.outputs.filtered-matrix) }} name: >- - ${{ matrix.config.model-prefix }} ${{ matrix.config.precision }} ${{ matrix.config.runner }} ${{ matrix.config.framework }} + p${{ matrix.config.priority }} | ${{ matrix.config.model-prefix }} ${{ matrix.config.precision }} ${{ matrix.config.runner }} ${{ matrix.config.framework }} TP${{ matrix.config.tp }}${{ matrix.config.pp != '' && matrix.config.pp != '1' && format('/PP{0}', matrix.config.pp) || '' }}${{ matrix.config.dcp-size != '' && matrix.config.dcp-size != '1' && format('/DCP{0}', matrix.config.dcp-size) || '' }}${{ matrix.config.pcp-size != '' && matrix.config.pcp-size != '1' && format('/PCP{0}', matrix.config.pcp-size) || '' }} c${{ matrix.config.conc }} - runs-on: ${{ matrix.config.runner }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + format( + '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + matrix.config.runner, + matrix.config.priority, + matrix.config['queue-token'] + ) || + format('["{0}"]', matrix.config.runner) + ) }} env: EXP_NAME: ${{ matrix.config.exp-name }} MODEL: ${{ matrix.config.model }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index b983982af8..41f3e6f8b7 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -212,6 +212,7 @@ jobs: - id: setup env: GH_TOKEN: ${{ github.token }} + PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} TRIM_CONC: >- ${{ github.event_name == 'pull_request' && @@ -255,6 +256,11 @@ jobs: fi CONFIG_JSON=$("${CMD[@]}") + CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ + "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ + --event-name "${{ github.event_name }}" \ + --queue-namespace "${{ github.run_id }}" \ + --labels-json "$PR_LABELS") echo "search-space-config=$CONFIG_JSON" >> "$GITHUB_OUTPUT" python3 "${GITHUB_WORKSPACE}/utils/find_reusable_sweep_run.py" \ @@ -325,6 +331,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -365,6 +373,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -441,6 +451,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -499,6 +511,8 @@ jobs: with: exp-name: ${{ matrix.config.exp-name }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -549,6 +563,8 @@ jobs: osl: '0' max-model-len: '0' runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -609,6 +625,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -652,6 +670,8 @@ jobs: osl: ${{ matrix.config.osl }} max-model-len: ${{ matrix.config.max-model-len }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index 10c6ddd9ad..b6724de965 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -14,6 +14,11 @@ on: required: false type: string default: 'b300' + priority: + description: "Lower-is-sooner CI priority score" + required: false + type: string + default: '2.250' model: description: "HF model id (basename must be in launcher STAGED_MODELS for pre-staged local weights)" required: false @@ -104,9 +109,19 @@ env: jobs: collect-al: - runs-on: ${{ inputs.runner }} + runs-on: >- + ${{ fromJSON( + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && + format( + '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + inputs.runner, + inputs.priority, + github.run_id + ) || + format('["{0}"]', inputs.runner) + ) }} timeout-minutes: 600 - name: "SpeedBench AL matrix | ${{ inputs.category }} | mtp=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" + name: "p${{ inputs.priority }} | SpeedBench AL matrix | ${{ inputs.category }} | mtp=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" steps: - name: Resource cleanup (pre-run) run: &resource-cleanup | diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml new file mode 100644 index 0000000000..228e9ac265 --- /dev/null +++ b/configs/ci-priority.yaml @@ -0,0 +1,51 @@ +version: 1 + +# Lower scores run first. Scores remain human-readable in workflow job labels +# (for example, ci-priority-p0.750). +base-score: 5.0 +minimum-score: 0.0 +maximum-score: 999.0 + +adjustments: + event: + push: -2.0 + multi-node: -1.25 + agentic: -1.0 + eval-only: 0.5 + precision: + fp4: -0.75 + spec-decoding: + mtp: -0.75 + eagle: -0.75 + eagle3: -0.75 + framework-prefix: + sglang: -0.5 + vllm: -0.5 + dynamo-sglang: -0.5 + dynamo-vllm: -0.5 + model-prefix: + glm5: -0.75 + glm5.1: -0.75 + kimik2.5: -0.75 + dsv4: -0.75 + minimaxm3: -0.75 + qwen3.5: -0.75 + dsr1: -0.75 + +labels: + # Repository labels are maintainer-controlled. A matching override wins over + # every automatic adjustment. + override-pattern: '^ci-priority:p(?P[0-9]+(?:\.[0-9]+)?)$' + checklist-complete: + names: [ci-checklist-complete] + adjustment: -0.25 + patchwork: + names: [ci-patchwork, engine-patch] + score: 999.0 + waived-by: [ci-patchwork-waived] + +scheduler: + # Waiting jobs improve by this many score points per hour. This prevents + # ordinary work from starving without allowing patchwork jobs to jump the + # queue in a realistic time window. + aging-per-hour: 0.25 diff --git a/utils/ci_priority.py b/utils/ci_priority.py new file mode 100755 index 0000000000..9661e7af8d --- /dev/null +++ b/utils/ci_priority.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Assign deterministic CI priority scores to generated benchmark jobs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from dataclasses import dataclass +from decimal import Decimal, ROUND_HALF_UP +from pathlib import Path +from typing import Any, Iterable + +import yaml + +SCORE_QUANTUM = Decimal("0.001") + + +@dataclass(frozen=True) +class PriorityContext: + event_name: str = "" + labels: frozenset[str] = frozenset() + queue_namespace: str = "local" + + +def _decimal(value: Any) -> Decimal: + return Decimal(str(value)) + + +def load_policy(path: str | Path) -> dict[str, Any]: + with Path(path).open() as policy_file: + policy = yaml.safe_load(policy_file) + if policy.get("version") != 1: + raise ValueError(f"Unsupported CI priority policy version: {policy.get('version')}") + return policy + + +def _first_prefix_adjustment(value: str, adjustments: dict[str, Any]) -> Decimal: + for prefix, adjustment in adjustments.items(): + if value == prefix or value.startswith(f"{prefix}-"): + return _decimal(adjustment) + return Decimal(0) + + +def _label_override(labels: Iterable[str], policy: dict[str, Any]) -> Decimal | None: + pattern = re.compile(policy["labels"]["override-pattern"]) + matches = [] + for label in labels: + match = pattern.fullmatch(label) + if match: + matches.append(_decimal(match.group("score"))) + if len(matches) > 1: + raise ValueError("Multiple ci-priority override labels are not allowed") + return matches[0] if matches else None + + +def calculate_priority( + entry: dict[str, Any], + policy: dict[str, Any], + context: PriorityContext = PriorityContext(), +) -> Decimal: + """Return a lower-is-sooner priority score for one benchmark matrix entry.""" + override = _label_override(context.labels, policy) + minimum = _decimal(policy["minimum-score"]) + maximum = _decimal(policy["maximum-score"]) + if override is not None: + return min(max(override, minimum), maximum).quantize(SCORE_QUANTUM) + + patchwork = policy["labels"]["patchwork"] + patch_labels = set(patchwork["names"]) + waiver_labels = set(patchwork.get("waived-by", [])) + if context.labels & patch_labels and not context.labels & waiver_labels: + return _decimal(patchwork["score"]).quantize(SCORE_QUANTUM) + + adjustments = policy["adjustments"] + score = _decimal(policy["base-score"]) + score += _decimal(adjustments.get("event", {}).get(context.event_name, 0)) + + if entry.get("prefill") is not None: + score += _decimal(adjustments.get("multi-node", 0)) + if entry.get("scenario-type") == "agentic-coding": + score += _decimal(adjustments.get("agentic", 0)) + if entry.get("eval-only") is True: + score += _decimal(adjustments.get("eval-only", 0)) + + score += _decimal(adjustments.get("precision", {}).get(str(entry.get("precision", "")), 0)) + score += _decimal( + adjustments.get("spec-decoding", {}).get(str(entry.get("spec-decoding", "")), 0) + ) + score += _first_prefix_adjustment( + str(entry.get("framework", "")), adjustments.get("framework-prefix", {}) + ) + score += _decimal( + adjustments.get("model-prefix", {}).get(str(entry.get("model-prefix", "")), 0) + ) + + checklist = policy["labels"].get("checklist-complete", {}) + if context.labels & set(checklist.get("names", [])): + score += _decimal(checklist.get("adjustment", 0)) + + return min(max(score, minimum), maximum).quantize(SCORE_QUANTUM, ROUND_HALF_UP) + + +def format_priority(score: Decimal) -> str: + return f"{score:.3f}" + + +def queue_token(value: dict[str, Any], namespace: str, path: tuple[str, ...]) -> str: + canonical = json.dumps(value, sort_keys=True, separators=(",", ":")) + material = f"{namespace}:{'/'.join(path)}:{canonical}".encode() + return hashlib.sha256(material).hexdigest()[:20] + + +def annotate_jobs( + value: Any, + policy: dict[str, Any], + context: PriorityContext = PriorityContext(), + *, + _path: tuple[str, ...] = (), +) -> Any: + """Copy a generated matrix payload and add scheduling metadata to every job.""" + if isinstance(value, list): + return [ + annotate_jobs(item, policy, context, _path=(*_path, str(index))) + for index, item in enumerate(value) + ] + if not isinstance(value, dict): + return value + + annotated = { + key: annotate_jobs(item, policy, context, _path=(*_path, key)) + for key, item in value.items() + } + if "runner" in value and "framework" in value: + annotated["priority"] = format_priority(calculate_priority(value, policy, context)) + annotated["queue-token"] = queue_token(value, context.queue_namespace, _path) + return annotated + + +def _labels_from_json(raw_labels: str) -> frozenset[str]: + if not raw_labels: + return frozenset() + value = json.loads(raw_labels) + if value is None: + return frozenset() + if not isinstance(value, list) or not all(isinstance(label, str) for label in value): + raise ValueError("--labels-json must be a JSON array of strings") + return frozenset(value) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--policy", default="configs/ci-priority.yaml") + parser.add_argument("--event-name", default="") + parser.add_argument("--labels-json", default="[]") + parser.add_argument("--queue-namespace", default="local") + parser.add_argument( + "--input", + type=Path, + help="Read matrix JSON from this file instead of stdin", + ) + args = parser.parse_args() + + policy = load_policy(args.policy) + context = PriorityContext( + event_name=args.event_name, + labels=_labels_from_json(args.labels_json), + queue_namespace=args.queue_namespace, + ) + source = args.input.read_text() if args.input else sys.stdin.read() + payload = json.loads(source) + json.dump(annotate_jobs(payload, policy, context), sys.stdout, separators=(",", ":")) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/ci_priority_controller.py b/utils/ci_priority_controller.py new file mode 100755 index 0000000000..3619cf150b --- /dev/null +++ b/utils/ci_priority_controller.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Plan or apply priority labels for idle self-hosted GitHub Actions runners.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from datetime import datetime, timezone +from decimal import Decimal +from pathlib import Path +from typing import Any, Iterable + +import yaml + +PRIORITY_LABEL_RE = re.compile(r"^ci-priority-p(?P[0-9]+(?:\.[0-9]+)?)$") +QUEUE_LABEL_RE = re.compile(r"^ci-queue-(?P[a-zA-Z0-9._-]+)$") + + +def parse_timestamp(value: str | None) -> datetime: + if not value: + return datetime.now(timezone.utc) + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def priority_from_labels(labels: Iterable[str]) -> tuple[str, Decimal] | None: + matches = [] + for label in labels: + match = PRIORITY_LABEL_RE.fullmatch(label) + if match: + matches.append((label, Decimal(match.group("score")))) + if len(matches) > 1: + raise ValueError(f"Job has multiple CI priority labels: {[label for label, _ in matches]}") + return matches[0] if matches else None + + +def queue_label_from_labels(labels: Iterable[str]) -> str | None: + matches = [label for label in labels if QUEUE_LABEL_RE.fullmatch(label)] + if len(matches) > 1: + raise ValueError(f"Job has multiple CI queue labels: {matches}") + return matches[0] if matches else None + + +def without_scheduling_labels(labels: Iterable[str]) -> frozenset[str]: + return frozenset( + label + for label in labels + if not PRIORITY_LABEL_RE.fullmatch(label) and not QUEUE_LABEL_RE.fullmatch(label) + ) + + +@dataclass(frozen=True) +class QueuedJob: + id: int + run_id: int + labels: frozenset[str] + queued_at: datetime + name: str = "" + + @classmethod + def from_payload(cls, payload: dict[str, Any], run_id: int | None = None) -> "QueuedJob": + return cls( + id=int(payload["id"]), + run_id=int(run_id if run_id is not None else payload.get("run_id", 0)), + labels=frozenset(payload.get("labels", [])), + queued_at=parse_timestamp(payload.get("created_at") or payload.get("queued_at")), + name=str(payload.get("name", "")), + ) + + +@dataclass(frozen=True) +class Runner: + id: int + name: str + labels: frozenset[str] + status: str + busy: bool + + @classmethod + def from_payload(cls, payload: dict[str, Any]) -> "Runner": + raw_labels = payload.get("labels", []) + labels = [item["name"] if isinstance(item, dict) else item for item in raw_labels] + return cls( + id=int(payload["id"]), + name=str(payload["name"]), + labels=frozenset(labels), + status=str(payload.get("status", "offline")), + busy=bool(payload.get("busy", False)), + ) + + +@dataclass(frozen=True) +class LabelUpdate: + runner_id: int + runner_name: str + labels: tuple[str, ...] + assigned_job_id: int | None + + def as_dict(self) -> dict[str, Any]: + return { + "runner_id": self.runner_id, + "runner_name": self.runner_name, + "labels": list(self.labels), + "assigned_job_id": self.assigned_job_id, + } + + +def effective_priority( + job: QueuedJob, + *, + now: datetime, + aging_per_hour: Decimal, +) -> Decimal: + priority = priority_from_labels(job.labels) + if priority is None: + raise ValueError(f"Job {job.id} has no CI priority label") + _, score = priority + waited_hours = Decimal(str(max(0.0, (now - job.queued_at).total_seconds()) / 3600)) + return max(Decimal(0), score - waited_hours * aging_per_hour) + + +def plan_label_updates( + jobs: Iterable[QueuedJob], + runners: Iterable[Runner], + *, + aging_per_hour: Decimal = Decimal("0.25"), + now: datetime | None = None, +) -> list[LabelUpdate]: + """Assign each idle runner to the best compatible queued job. + + Busy and offline runners are never relabeled. Priority and one-shot queue + labels are removed from unassigned idle runners, so a runner cannot take a + second job before the controller has considered newly queued higher + priorities. + """ + now = now or datetime.now(timezone.utc) + eligible_jobs = [ + job + for job in jobs + if priority_from_labels(job.labels) is not None + and queue_label_from_labels(job.labels) is not None + ] + eligible_jobs.sort( + key=lambda job: ( + effective_priority(job, now=now, aging_per_hour=aging_per_hour), + job.queued_at, + job.id, + ) + ) + + idle_runners = [runner for runner in runners if runner.status == "online" and not runner.busy] + remaining = {runner.id: runner for runner in idle_runners} + desired: dict[int, tuple[frozenset[str], int | None]] = { + runner.id: (without_scheduling_labels(runner.labels), None) + for runner in idle_runners + } + + for job in eligible_jobs: + priority = priority_from_labels(job.labels) + queue_label = queue_label_from_labels(job.labels) + if priority is None or queue_label is None: + continue + priority_label, _ = priority + required = without_scheduling_labels(job.labels) + candidates = [ + runner + for runner in remaining.values() + if required.issubset(without_scheduling_labels(runner.labels)) + ] + if not candidates: + continue + runner = min(candidates, key=lambda candidate: (candidate.name, candidate.id)) + desired[runner.id] = ( + without_scheduling_labels(runner.labels) | {priority_label, queue_label}, + job.id, + ) + del remaining[runner.id] + + updates = [] + for runner in sorted(idle_runners, key=lambda item: (item.name, item.id)): + labels, job_id = desired[runner.id] + if labels != runner.labels: + updates.append( + LabelUpdate( + runner_id=runner.id, + runner_name=runner.name, + labels=tuple(sorted(labels)), + assigned_job_id=job_id, + ) + ) + return updates + + +class GitHubClient: + def __init__(self, repository: str, token: str, api_url: str = "https://api.github.com"): + if repository.count("/") != 1: + raise ValueError("Repository must be OWNER/REPO") + self.repository = repository + self.token = token + self.api_url = api_url.rstrip("/") + + def request(self, method: str, path: str, body: Any = None) -> Any: + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request( + f"{self.api_url}{path}", + data=data, + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + raw = response.read() + except urllib.error.HTTPError as error: + detail = error.read().decode(errors="replace") + raise RuntimeError(f"GitHub API {method} {path} failed: {error.code} {detail}") from error + return json.loads(raw) if raw else None + + def paged(self, path: str, key: str) -> list[dict[str, Any]]: + separator = "&" if "?" in path else "?" + values = [] + page = 1 + while True: + payload = self.request("GET", f"{path}{separator}per_page=100&page={page}") + batch = payload[key] + values.extend(batch) + if len(batch) < 100: + return values + page += 1 + + def queued_jobs(self) -> list[QueuedJob]: + base = f"/repos/{self.repository}" + runs_by_id = {} + for status in ("queued", "in_progress"): + runs = self.paged( + f"{base}/actions/runs?status={status}", + "workflow_runs", + ) + runs_by_id.update({int(run["id"]): run for run in runs}) + + jobs = [] + for run_id in sorted(runs_by_id): + run_jobs = self.paged( + f"{base}/actions/runs/{run_id}/jobs?filter=latest", + "jobs", + ) + jobs.extend( + QueuedJob.from_payload(job, run_id) + for job in run_jobs + if job.get("status") == "queued" + ) + return jobs + + def runners(self) -> list[Runner]: + payloads = self.paged(f"/repos/{self.repository}/actions/runners", "runners") + return [Runner.from_payload(payload) for payload in payloads] + + def replace_runner_labels(self, runner_id: int, labels: Iterable[str]) -> None: + self.request( + "PUT", + f"/repos/{self.repository}/actions/runners/{runner_id}/labels", + {"labels": sorted(labels)}, + ) + + +def load_aging_rate(policy_path: str | Path) -> Decimal: + with Path(policy_path).open() as policy_file: + policy = yaml.safe_load(policy_file) + return Decimal(str(policy["scheduler"]["aging-per-hour"])) + + +def reconcile(client: GitHubClient, aging_per_hour: Decimal, apply: bool) -> list[LabelUpdate]: + updates = plan_label_updates( + client.queued_jobs(), + client.runners(), + aging_per_hour=aging_per_hour, + ) + if apply: + for update in updates: + client.replace_runner_labels(update.runner_id, update.labels) + return updates + + +def _load_json(path: Path) -> Any: + with path.open() as source: + return json.load(source) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--policy", default="configs/ci-priority.yaml") + subparsers = parser.add_subparsers(dest="command", required=True) + + plan_parser = subparsers.add_parser("plan", help="Plan from local API response fixtures") + plan_parser.add_argument("--jobs", type=Path, required=True) + plan_parser.add_argument("--runners", type=Path, required=True) + plan_parser.add_argument("--now", help="ISO-8601 clock for deterministic simulations") + + reconcile_parser = subparsers.add_parser("reconcile", help="Poll GitHub and reconcile runner labels") + reconcile_parser.add_argument("--repository", required=True) + reconcile_parser.add_argument("--api-url", default="https://api.github.com") + reconcile_parser.add_argument("--token-env", default="GITHUB_TOKEN") + reconcile_parser.add_argument("--apply", action="store_true") + reconcile_parser.add_argument("--watch", type=float, default=0, metavar="SECONDS") + + args = parser.parse_args() + aging_per_hour = load_aging_rate(args.policy) + + if args.command == "plan": + raw_jobs = _load_json(args.jobs) + raw_runners = _load_json(args.runners) + jobs = [QueuedJob.from_payload(job) for job in raw_jobs] + runners = [Runner.from_payload(runner) for runner in raw_runners] + now = parse_timestamp(args.now) if args.now else None + updates = plan_label_updates(jobs, runners, aging_per_hour=aging_per_hour, now=now) + json.dump([update.as_dict() for update in updates], sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + token = os.environ.get(args.token_env) + if not token: + parser.error(f"{args.token_env} must contain a GitHub token") + client = GitHubClient(args.repository, token, args.api_url) + while True: + updates = reconcile(client, aging_per_hour, args.apply) + print(json.dumps([update.as_dict() for update in updates], separators=(",", ":"))) + if args.watch <= 0: + return 0 + time.sleep(args.watch) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py new file mode 100644 index 0000000000..955db3fc17 --- /dev/null +++ b/utils/test_ci_priority.py @@ -0,0 +1,99 @@ +from decimal import Decimal +from pathlib import Path + +import pytest + +from ci_priority import PriorityContext, annotate_jobs, calculate_priority, load_policy + + +POLICY = load_policy(Path(__file__).parents[1] / "configs" / "ci-priority.yaml") + + +def test_combined_high_value_signals_outrank_baseline_job(): + baseline = { + "runner": "h100", + "framework": "trt", + "model-prefix": "other", + "precision": "fp8", + "spec-decoding": "none", + } + high_value = { + **baseline, + "runner": "b200-multinode", + "framework": "sglang", + "model-prefix": "dsv4", + "precision": "fp4", + "spec-decoding": "mtp", + "scenario-type": "agentic-coding", + "prefill": {"hardware": "b200"}, + "decode": {"hardware": "b200"}, + } + + assert calculate_priority(high_value, POLICY) == Decimal("0.000") + assert calculate_priority(baseline, POLICY) == Decimal("5.000") + + +def test_main_branch_jobs_receive_an_automatic_boost(): + entry = {"runner": "h100", "framework": "trt"} + + assert calculate_priority( + entry, + POLICY, + PriorityContext(event_name="push"), + ) == Decimal("3.000") + + +def test_patchwork_label_forces_bottom_priority_without_waiver(): + entry = {"runner": "b200", "framework": "sglang", "precision": "fp4"} + + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"ci-patchwork"})), + ) == Decimal("999.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"ci-patchwork", "ci-patchwork-waived"})), + ) < Decimal("999.000") + + +def test_maintainer_override_wins_and_conflicts_are_rejected(): + entry = {"runner": "h100", "framework": "trt"} + + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"ci-priority:p0.7"})), + ) == Decimal("0.700") + with pytest.raises(ValueError, match="Multiple ci-priority override"): + calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"ci-priority:p0.7", "ci-priority:p2.5"})), + ) + + +def test_annotation_only_touches_runnable_matrix_entries(): + payload = { + "single_node": { + "1k1k": [ + { + "runner": "b200", + "framework": "sglang", + "model-prefix": "qwen3.5", + "precision": "fp4", + "spec-decoding": "mtp", + } + ] + }, + "changelog_metadata": {"runner": "not-a-job"}, + } + + annotated = annotate_jobs(payload, POLICY) + + assert annotated["single_node"]["1k1k"][0]["priority"] == "2.250" + assert len(annotated["single_node"]["1k1k"][0]["queue-token"]) == 20 + assert "priority" not in annotated["changelog_metadata"] + assert "priority" not in payload["single_node"]["1k1k"][0] + assert "queue-token" not in payload["single_node"]["1k1k"][0] diff --git a/utils/test_ci_priority_controller.py b/utils/test_ci_priority_controller.py new file mode 100644 index 0000000000..3f21010279 --- /dev/null +++ b/utils/test_ci_priority_controller.py @@ -0,0 +1,151 @@ +from datetime import datetime, timedelta, timezone +from decimal import Decimal + +from ci_priority_controller import GitHubClient, QueuedJob, Runner, plan_label_updates + + +NOW = datetime(2026, 7, 14, 18, 0, tzinfo=timezone.utc) + + +def job(job_id, priority, hardware, queued_minutes=0): + return QueuedJob( + id=job_id, + run_id=100 + job_id, + labels=frozenset({ + "self-hosted", + hardware, + f"ci-priority-p{priority}", + f"ci-queue-job-{job_id}", + }), + queued_at=NOW - timedelta(minutes=queued_minutes), + name=f"job-{job_id}", + ) + + +def runner(runner_id, name, *labels, busy=False, status="online"): + return Runner( + id=runner_id, + name=name, + labels=frozenset({"self-hosted", "Linux", "X64", name, *labels}), + status=status, + busy=busy, + ) + + +def test_assigns_best_compatible_job_to_each_idle_runner(): + jobs = [ + job(1, "5.000", "h100", queued_minutes=30), + job(2, "0.700", "b200", queued_minutes=1), + job(3, "2.500", "h100", queued_minutes=2), + ] + runners = [ + runner(10, "b200_00", "b200"), + runner(11, "h100_00", "h100"), + ] + + updates = plan_label_updates(jobs, runners, now=NOW) + + assert [(update.runner_name, update.assigned_job_id) for update in updates] == [ + ("b200_00", 2), + ("h100_00", 3), + ] + assert "ci-priority-p0.700" in updates[0].labels + assert "ci-priority-p2.500" in updates[1].labels + assert "ci-queue-job-2" in updates[0].labels + assert "ci-queue-job-3" in updates[1].labels + + +def test_aging_breaks_starvation_between_nearby_priorities(): + jobs = [ + job(1, "3.000", "h100", queued_minutes=8 * 60), + job(2, "1.500", "h100", queued_minutes=1), + ] + runners = [runner(11, "h100_00", "h100")] + + updates = plan_label_updates( + jobs, + runners, + now=NOW, + aging_per_hour=Decimal("0.25"), + ) + + assert updates[0].assigned_job_id == 1 + + +def test_unused_idle_runner_loses_stale_priority_label(): + runners = [ + runner( + 10, + "b200_00", + "b200", + "ci-priority-p5.000", + "ci-queue-old-job", + ) + ] + + updates = plan_label_updates([], runners, now=NOW) + + assert len(updates) == 1 + assert updates[0].assigned_job_id is None + assert all(not label.startswith("ci-priority-p") for label in updates[0].labels) + assert all(not label.startswith("ci-queue-") for label in updates[0].labels) + + +def test_busy_and_offline_runners_are_never_relabelled(): + runners = [ + runner(10, "b200_00", "b200", "ci-priority-p5.000", busy=True), + runner(11, "b200_01", "b200", "ci-priority-p5.000", status="offline"), + ] + + assert plan_label_updates([job(1, "0.700", "b200")], runners, now=NOW) == [] + + +def test_exact_runner_name_remains_a_compatibility_constraint(): + jobs = [job(1, "1.000", "h100_01")] + runners = [ + runner(10, "h100_00", "h100"), + runner(11, "h100_01", "h100"), + ] + + updates = plan_label_updates(jobs, runners, now=NOW) + + assigned = [update for update in updates if update.assigned_job_id is not None] + assert len(assigned) == 1 + assert assigned[0].runner_name == "h100_01" + + +def test_discovers_queued_jobs_inside_in_progress_workflow_runs(): + class FixtureClient(GitHubClient): + def __init__(self): + super().__init__("owner/repo", "unused") + self.paths = [] + + def paged(self, path, key): + self.paths.append(path) + if "status=queued" in path: + return [{"id": 1}] + if "status=in_progress" in path: + return [{"id": 2}] + if "/runs/1/jobs" in path: + return [{"id": 11, "status": "completed", "labels": []}] + if "/runs/2/jobs" in path: + return [{ + "id": 22, + "status": "queued", + "created_at": "2026-07-14T18:00:00Z", + "labels": [ + "self-hosted", + "b200", + "ci-priority-p1.000", + "ci-queue-job-22", + ], + }] + raise AssertionError(path) + + client = FixtureClient() + + jobs = client.queued_jobs() + + assert [queued_job.id for queued_job in jobs] == [22] + assert any("status=queued" in path for path in client.paths) + assert any("status=in_progress" in path for path in client.paths) From 261c631f4d9f1901878abd0c459951f1aa7bfa50 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:51:17 -0500 Subject: [PATCH 02/21] Reserve p0 for Core skip queue --- .github/workflows/run-sweep.yml | 34 +++++++- configs/ci-priority.yaml | 9 ++- utils/authorize_skip_queue.py | 125 +++++++++++++++++++++++++++++ utils/ci_priority.py | 9 +++ utils/test_authorize_skip_queue.py | 78 ++++++++++++++++++ utils/test_ci_priority.py | 30 ++++++- 6 files changed, 280 insertions(+), 5 deletions(-) create mode 100755 utils/authorize_skip_queue.py create mode 100644 utils/test_authorize_skip_queue.py diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 41f3e6f8b7..2a5fd7d677 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -13,6 +13,7 @@ concurrency: github.event.label.name != 'full-sweep-fail-fast-no-canary' && github.event.label.name != 'all-evals' && github.event.label.name != 'evals-only' && + github.event.label.name != 'skip_queue' && github.run_id || 'active' }} @@ -49,7 +50,8 @@ jobs: github.event.label.name == 'full-sweep-fail-fast' || github.event.label.name == 'full-sweep-fail-fast-no-canary' || github.event.label.name == 'all-evals' || - github.event.label.name == 'evals-only' + github.event.label.name == 'evals-only' || + github.event.label.name == 'skip_queue' ) outputs: skip-pr-sweep: ${{ steps.sweep_policy.outputs.skip-pr-sweep }} @@ -155,6 +157,11 @@ jobs: setup: needs: [check-changelog, reuse-sweep-gate] runs-on: ubuntu-latest + permissions: + actions: read + contents: read + issues: read + pull-requests: read if: >- always() && ( @@ -188,7 +195,8 @@ jobs: github.event.label.name == 'full-sweep-fail-fast' || github.event.label.name == 'full-sweep-fail-fast-no-canary' || github.event.label.name == 'all-evals' || - github.event.label.name == 'evals-only' + github.event.label.name == 'evals-only' || + github.event.label.name == 'skip_queue' ) ) || ( @@ -209,10 +217,24 @@ jobs: with: fetch-depth: 0 + - name: Authorize skip_queue + id: authorize_skip_queue + if: >- + github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'skip_queue') + env: + CORE_TEAM_READ_TOKEN: ${{ secrets.CORE_TEAM_READ_TOKEN }} + run: | + authorized=$(python3 "${GITHUB_WORKSPACE}/utils/authorize_skip_queue.py" \ + --repository "${{ github.repository }}" \ + --pr-number "${{ github.event.pull_request.number }}") + echo "authorized=$authorized" >> "$GITHUB_OUTPUT" + - id: setup env: GH_TOKEN: ${{ github.token }} PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} + SKIP_QUEUE_AUTHORIZED: ${{ steps.authorize_skip_queue.outputs.authorized == 'true' }} TRIM_CONC: >- ${{ github.event_name == 'pull_request' && @@ -255,12 +277,18 @@ jobs: CMD+=(--evals-only) fi + PRIORITY_ARGS=() + if [ "$SKIP_QUEUE_AUTHORIZED" = "true" ]; then + PRIORITY_ARGS+=(--skip-queue-authorized) + fi + CONFIG_JSON=$("${CMD[@]}") CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ --event-name "${{ github.event_name }}" \ --queue-namespace "${{ github.run_id }}" \ - --labels-json "$PR_LABELS") + --labels-json "$PR_LABELS" \ + "${PRIORITY_ARGS[@]}") echo "search-space-config=$CONFIG_JSON" >> "$GITHUB_OUTPUT" python3 "${GITHUB_WORKSPACE}/utils/find_reusable_sweep_run.py" \ diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index 228e9ac265..e1cdd34d7f 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -3,7 +3,7 @@ version: 1 # Lower scores run first. Scores remain human-readable in workflow job labels # (for example, ci-priority-p0.750). base-score: 5.0 -minimum-score: 0.0 +minimum-score: 0.1 maximum-score: 999.0 adjustments: @@ -36,6 +36,13 @@ labels: # Repository labels are maintainer-controlled. A matching override wins over # every automatic adjustment. override-pattern: '^ci-priority:p(?P[0-9]+(?:\.[0-9]+)?)$' + # p0 is reserved for an active skip_queue label whose latest labeling actor + # is verified as an active SemiAnalysisAI/Core team member. + skip-queue: + name: skip_queue + score: 0.0 + organization: SemiAnalysisAI + team-slug: core checklist-complete: names: [ci-checklist-complete] adjustment: -0.25 diff --git a/utils/authorize_skip_queue.py b/utils/authorize_skip_queue.py new file mode 100755 index 0000000000..c509457d2d --- /dev/null +++ b/utils/authorize_skip_queue.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Verify that the active skip_queue label was applied by a trusted team member.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from typing import Any + + +class GitHubApi: + def __init__(self, token: str, api_url: str = "https://api.github.com"): + self.token = token + self.api_url = api_url.rstrip("/") + + def request(self, path: str) -> Any: + request = urllib.request.Request( + f"{self.api_url}{path}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read()) + except urllib.error.HTTPError as error: + detail = error.read().decode(errors="replace") + raise RuntimeError(f"GitHub API GET {path} failed: {error.code} {detail}") from error + + def paged(self, path: str) -> list[dict[str, Any]]: + separator = "&" if "?" in path else "?" + values = [] + page = 1 + while True: + batch = self.request(f"{path}{separator}per_page=100&page={page}") + values.extend(batch) + if len(batch) < 100: + return values + page += 1 + + +def active_label_actor(events: list[dict[str, Any]], label_name: str) -> str | None: + """Return who applied the currently active instance of a label.""" + actor = None + for event in events: + if event.get("label", {}).get("name") != label_name: + continue + if event.get("event") == "labeled": + actor = event.get("actor", {}).get("login") + elif event.get("event") == "unlabeled": + actor = None + return actor + + +def is_authorized( + api: GitHubApi, + *, + repository: str, + pr_number: int, + organization: str, + team_slug: str, + label_name: str, +) -> tuple[bool, str | None]: + events = api.paged(f"/repos/{repository}/issues/{pr_number}/timeline") + actor = active_label_actor(events, label_name) + if not actor: + return False, None + + membership = api.request( + f"/orgs/{organization}/teams/{team_slug}/memberships/{actor}" + ) + return membership.get("state") == "active", actor + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", required=True) + parser.add_argument("--pr-number", required=True, type=int) + parser.add_argument("--organization", default="SemiAnalysisAI") + parser.add_argument("--team-slug", default="core") + parser.add_argument("--label", default="skip_queue") + parser.add_argument("--token-env", default="CORE_TEAM_READ_TOKEN") + parser.add_argument("--api-url", default="https://api.github.com") + args = parser.parse_args() + + token = os.environ.get(args.token_env) + if not token: + print(f"::warning::{args.token_env} is unavailable; refusing skip_queue authorization", file=sys.stderr) + print("false") + return 0 + + try: + authorized, actor = is_authorized( + GitHubApi(token, args.api_url), + repository=args.repository, + pr_number=args.pr_number, + organization=args.organization, + team_slug=args.team_slug, + label_name=args.label, + ) + except RuntimeError as error: + print(f"::warning::{error}; refusing skip_queue authorization", file=sys.stderr) + print("false") + return 0 + + if authorized: + print(f"::notice::skip_queue authorized by {actor}", file=sys.stderr) + elif actor: + print( + f"::warning::skip_queue was applied by {actor}, who is not an active " + f"{args.organization}/{args.team_slug} member", + file=sys.stderr, + ) + print("true" if authorized else "false") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/ci_priority.py b/utils/ci_priority.py index 9661e7af8d..b8d034f8ca 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -23,6 +23,7 @@ class PriorityContext: event_name: str = "" labels: frozenset[str] = frozenset() queue_namespace: str = "local" + skip_queue_authorized: bool = False def _decimal(value: Any) -> Decimal: @@ -65,6 +66,12 @@ def calculate_priority( override = _label_override(context.labels, policy) minimum = _decimal(policy["minimum-score"]) maximum = _decimal(policy["maximum-score"]) + skip_queue = policy["labels"]["skip-queue"] + if ( + context.skip_queue_authorized + and skip_queue["name"] in context.labels + ): + return _decimal(skip_queue["score"]).quantize(SCORE_QUANTUM) if override is not None: return min(max(override, minimum), maximum).quantize(SCORE_QUANTUM) @@ -156,6 +163,7 @@ def main() -> int: parser.add_argument("--event-name", default="") parser.add_argument("--labels-json", default="[]") parser.add_argument("--queue-namespace", default="local") + parser.add_argument("--skip-queue-authorized", action="store_true") parser.add_argument( "--input", type=Path, @@ -168,6 +176,7 @@ def main() -> int: event_name=args.event_name, labels=_labels_from_json(args.labels_json), queue_namespace=args.queue_namespace, + skip_queue_authorized=args.skip_queue_authorized, ) source = args.input.read_text() if args.input else sys.stdin.read() payload = json.loads(source) diff --git a/utils/test_authorize_skip_queue.py b/utils/test_authorize_skip_queue.py new file mode 100644 index 0000000000..f0f8c5066a --- /dev/null +++ b/utils/test_authorize_skip_queue.py @@ -0,0 +1,78 @@ +from authorize_skip_queue import active_label_actor, is_authorized + + +class FakeApi: + def __init__(self, events, memberships): + self.events = events + self.memberships = memberships + self.membership_requests = [] + + def paged(self, path): + assert path == "/repos/SemiAnalysisAI/InferenceX/issues/42/timeline" + return self.events + + def request(self, path): + actor = path.rsplit("/", 1)[-1] + self.membership_requests.append(actor) + return self.memberships.get(actor, {"state": "inactive"}) + + +def labeled(actor): + return { + "event": "labeled", + "label": {"name": "skip_queue"}, + "actor": {"login": actor}, + } + + +def unlabeled(actor): + return { + "event": "unlabeled", + "label": {"name": "skip_queue"}, + "actor": {"login": actor}, + } + + +def authorize(api): + return is_authorized( + api, + repository="SemiAnalysisAI/InferenceX", + pr_number=42, + organization="SemiAnalysisAI", + team_slug="core", + label_name="skip_queue", + ) + + +def test_active_label_from_active_core_member_is_authorized(): + api = FakeApi([labeled("alice")], {"alice": {"state": "active"}}) + + assert authorize(api) == (True, "alice") + + +def test_active_label_from_nonmember_is_rejected(): + api = FakeApi([labeled("mallory")], {"mallory": {"state": "inactive"}}) + + assert authorize(api) == (False, "mallory") + + +def test_latest_labeling_actor_controls_after_remove_and_readd(): + events = [labeled("alice"), unlabeled("alice"), labeled("mallory")] + api = FakeApi( + events, + { + "alice": {"state": "active"}, + "mallory": {"state": "inactive"}, + }, + ) + + assert active_label_actor(events, "skip_queue") == "mallory" + assert authorize(api) == (False, "mallory") + assert api.membership_requests == ["mallory"] + + +def test_removed_label_is_not_authorized_or_looked_up(): + api = FakeApi([labeled("alice"), unlabeled("bob")], {"alice": {"state": "active"}}) + + assert authorize(api) == (False, None) + assert api.membership_requests == [] diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 955db3fc17..0602576361 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -29,7 +29,7 @@ def test_combined_high_value_signals_outrank_baseline_job(): "decode": {"hardware": "b200"}, } - assert calculate_priority(high_value, POLICY) == Decimal("0.000") + assert calculate_priority(high_value, POLICY) == Decimal("0.100") assert calculate_priority(baseline, POLICY) == Decimal("5.000") @@ -43,6 +43,29 @@ def test_main_branch_jobs_receive_an_automatic_boost(): ) == Decimal("3.000") +def test_p0_requires_both_skip_queue_label_and_core_authorization(): + entry = {"runner": "h100", "framework": "trt"} + + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"skip_queue"})), + ) == Decimal("5.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext(skip_queue_authorized=True), + ) == Decimal("5.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext( + labels=frozenset({"skip_queue"}), + skip_queue_authorized=True, + ), + ) == Decimal("0.000") + + def test_patchwork_label_forces_bottom_priority_without_waiver(): entry = {"runner": "b200", "framework": "sglang", "precision": "fp4"} @@ -66,6 +89,11 @@ def test_maintainer_override_wins_and_conflicts_are_rejected(): POLICY, PriorityContext(labels=frozenset({"ci-priority:p0.7"})), ) == Decimal("0.700") + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"ci-priority:p0"})), + ) == Decimal("0.100") with pytest.raises(ValueError, match="Multiple ci-priority override"): calculate_priority( entry, From 3950d26c6f116df4821d83bd96c6ba858a26a771 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:58:44 -0500 Subject: [PATCH 03/21] Make CI priorities higher-first --- .../workflows/benchmark-multinode-tmpl.yml | 4 +- .github/workflows/benchmark-tmpl.yml | 4 +- .github/workflows/collectivex-sweep.yml | 4 +- .github/workflows/speedbench-al.yml | 4 +- configs/ci-priority.yaml | 58 +++++++++---------- utils/ci_priority.py | 14 ++--- utils/ci_priority_controller.py | 12 ++-- utils/test_ci_priority.py | 38 ++++++++---- utils/test_ci_priority_controller.py | 23 ++++++-- 9 files changed, 95 insertions(+), 66 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 0c45621417..3bcb26eb82 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -7,10 +7,10 @@ on: required: true type: string priority: - description: "Lower-is-sooner CI priority score" + description: "Higher-is-sooner CI priority score" required: false type: string - default: '5.000' + default: '0.000' queue-token: description: "One-shot queue token generated for this job" required: false diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 6d854989dc..4ce501d2b4 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -6,10 +6,10 @@ on: required: true type: string priority: - description: "Lower-is-sooner CI priority score" + description: "Higher-is-sooner CI priority score" required: false type: string - default: '5.000' + default: '0.000' queue-token: description: "One-shot queue token generated for this job" required: false diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml index 1ba2d9cfc9..0351e4fd6b 100644 --- a/.github/workflows/collectivex-sweep.yml +++ b/.github/workflows/collectivex-sweep.yml @@ -24,9 +24,9 @@ on: type: string default: '' priority: - description: Lower-is-sooner CI priority score + description: Higher-is-sooner CI priority score type: string - default: '5.000' + default: '0.000' concurrency: group: cx-${{ github.ref }}-${{ inputs.backend }}-${{ inputs.only_sku }} cancel-in-progress: false diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index b6724de965..968ca0816a 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -15,10 +15,10 @@ on: type: string default: 'b300' priority: - description: "Lower-is-sooner CI priority score" + description: "Higher-is-sooner CI priority score" required: false type: string - default: '2.250' + default: '2.750' model: description: "HF model id (basename must be in launcher STAGED_MODELS for pre-staged local weights)" required: false diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index e1cdd34d7f..aba03d629e 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -1,54 +1,52 @@ version: 1 -# Lower scores run first. Scores remain human-readable in workflow job labels -# (for example, ci-priority-p0.750). -base-score: 5.0 -minimum-score: 0.1 -maximum-score: 999.0 +# Higher scores run first. Scores are uncapped and remain human-readable in +# workflow job labels (for example, ci-priority-p2.750). +base-score: 0.0 adjustments: event: - push: -2.0 - multi-node: -1.25 - agentic: -1.0 - eval-only: 0.5 + push: 2.0 + multi-node: 1.25 + agentic: 1.0 + eval-only: -0.5 precision: - fp4: -0.75 + fp4: 0.75 spec-decoding: - mtp: -0.75 - eagle: -0.75 - eagle3: -0.75 + mtp: 0.75 + eagle: 0.75 + eagle3: 0.75 framework-prefix: - sglang: -0.5 - vllm: -0.5 - dynamo-sglang: -0.5 - dynamo-vllm: -0.5 + sglang: 0.5 + vllm: 0.5 + dynamo-sglang: 0.5 + dynamo-vllm: 0.5 model-prefix: - glm5: -0.75 - glm5.1: -0.75 - kimik2.5: -0.75 - dsv4: -0.75 - minimaxm3: -0.75 - qwen3.5: -0.75 - dsr1: -0.75 + glm5: 0.75 + glm5.1: 0.75 + kimik2.5: 0.75 + dsv4: 0.75 + minimaxm3: 0.75 + qwen3.5: 0.75 + dsr1: 0.75 labels: # Repository labels are maintainer-controlled. A matching override wins over # every automatic adjustment. - override-pattern: '^ci-priority:p(?P[0-9]+(?:\.[0-9]+)?)$' - # p0 is reserved for an active skip_queue label whose latest labeling actor - # is verified as an active SemiAnalysisAI/Core team member. + override-pattern: '^ci-priority:p(?P-?[0-9]+(?:\.[0-9]+)?)$' + # pskip is reserved for an active skip_queue label whose latest labeling + # actor is verified as an active SemiAnalysisAI/Core team member. skip-queue: name: skip_queue - score: 0.0 + score: skip organization: SemiAnalysisAI team-slug: core checklist-complete: names: [ci-checklist-complete] - adjustment: -0.25 + adjustment: 0.25 patchwork: names: [ci-patchwork, engine-patch] - score: 999.0 + score: -999.0 waived-by: [ci-patchwork-waived] scheduler: diff --git a/utils/ci_priority.py b/utils/ci_priority.py index b8d034f8ca..d86ae4e601 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -62,18 +62,18 @@ def calculate_priority( policy: dict[str, Any], context: PriorityContext = PriorityContext(), ) -> Decimal: - """Return a lower-is-sooner priority score for one benchmark matrix entry.""" + """Return a higher-is-sooner priority score for one benchmark matrix entry.""" override = _label_override(context.labels, policy) - minimum = _decimal(policy["minimum-score"]) - maximum = _decimal(policy["maximum-score"]) skip_queue = policy["labels"]["skip-queue"] if ( context.skip_queue_authorized and skip_queue["name"] in context.labels ): - return _decimal(skip_queue["score"]).quantize(SCORE_QUANTUM) + if skip_queue["score"] != "skip": + raise ValueError("Authorized skip_queue priority must use the 'skip' sentinel") + return Decimal("Infinity") if override is not None: - return min(max(override, minimum), maximum).quantize(SCORE_QUANTUM) + return override.quantize(SCORE_QUANTUM) patchwork = policy["labels"]["patchwork"] patch_labels = set(patchwork["names"]) @@ -107,11 +107,11 @@ def calculate_priority( if context.labels & set(checklist.get("names", [])): score += _decimal(checklist.get("adjustment", 0)) - return min(max(score, minimum), maximum).quantize(SCORE_QUANTUM, ROUND_HALF_UP) + return score.quantize(SCORE_QUANTUM, ROUND_HALF_UP) def format_priority(score: Decimal) -> str: - return f"{score:.3f}" + return "skip" if score.is_infinite() else f"{score:.3f}" def queue_token(value: dict[str, Any], namespace: str, path: tuple[str, ...]) -> str: diff --git a/utils/ci_priority_controller.py b/utils/ci_priority_controller.py index 3619cf150b..6fa84ee3ba 100755 --- a/utils/ci_priority_controller.py +++ b/utils/ci_priority_controller.py @@ -20,7 +20,9 @@ import yaml -PRIORITY_LABEL_RE = re.compile(r"^ci-priority-p(?P[0-9]+(?:\.[0-9]+)?)$") +PRIORITY_LABEL_RE = re.compile( + r"^ci-priority-p(?Pskip|-?[0-9]+(?:\.[0-9]+)?)$" +) QUEUE_LABEL_RE = re.compile(r"^ci-queue-(?P[a-zA-Z0-9._-]+)$") @@ -35,7 +37,9 @@ def priority_from_labels(labels: Iterable[str]) -> tuple[str, Decimal] | None: for label in labels: match = PRIORITY_LABEL_RE.fullmatch(label) if match: - matches.append((label, Decimal(match.group("score")))) + raw_score = match.group("score") + score = Decimal("Infinity") if raw_score == "skip" else Decimal(raw_score) + matches.append((label, score)) if len(matches) > 1: raise ValueError(f"Job has multiple CI priority labels: {[label for label, _ in matches]}") return matches[0] if matches else None @@ -123,7 +127,7 @@ def effective_priority( raise ValueError(f"Job {job.id} has no CI priority label") _, score = priority waited_hours = Decimal(str(max(0.0, (now - job.queued_at).total_seconds()) / 3600)) - return max(Decimal(0), score - waited_hours * aging_per_hour) + return score + waited_hours * aging_per_hour def plan_label_updates( @@ -149,7 +153,7 @@ def plan_label_updates( ] eligible_jobs.sort( key=lambda job: ( - effective_priority(job, now=now, aging_per_hour=aging_per_hour), + -effective_priority(job, now=now, aging_per_hour=aging_per_hour), job.queued_at, job.id, ) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 0602576361..18463086b5 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -29,8 +29,8 @@ def test_combined_high_value_signals_outrank_baseline_job(): "decode": {"hardware": "b200"}, } - assert calculate_priority(high_value, POLICY) == Decimal("0.100") - assert calculate_priority(baseline, POLICY) == Decimal("5.000") + assert calculate_priority(high_value, POLICY) == Decimal("5.000") + assert calculate_priority(baseline, POLICY) == Decimal("0.000") def test_main_branch_jobs_receive_an_automatic_boost(): @@ -40,22 +40,22 @@ def test_main_branch_jobs_receive_an_automatic_boost(): entry, POLICY, PriorityContext(event_name="push"), - ) == Decimal("3.000") + ) == Decimal("2.000") -def test_p0_requires_both_skip_queue_label_and_core_authorization(): - entry = {"runner": "h100", "framework": "trt"} +def test_skip_queue_requires_both_label_and_core_authorization(): + entry = {"runner": "h100", "framework": "sglang", "precision": "fp4"} assert calculate_priority( entry, POLICY, PriorityContext(labels=frozenset({"skip_queue"})), - ) == Decimal("5.000") + ) == Decimal("1.250") assert calculate_priority( entry, POLICY, PriorityContext(skip_queue_authorized=True), - ) == Decimal("5.000") + ) == Decimal("1.250") assert calculate_priority( entry, POLICY, @@ -63,7 +63,16 @@ def test_p0_requires_both_skip_queue_label_and_core_authorization(): labels=frozenset({"skip_queue"}), skip_queue_authorized=True, ), - ) == Decimal("0.000") + ) == Decimal("Infinity") + annotated = annotate_jobs( + [entry], + POLICY, + PriorityContext( + labels=frozenset({"skip_queue"}), + skip_queue_authorized=True, + ), + ) + assert annotated[0]["priority"] == "skip" def test_patchwork_label_forces_bottom_priority_without_waiver(): @@ -73,12 +82,12 @@ def test_patchwork_label_forces_bottom_priority_without_waiver(): entry, POLICY, PriorityContext(labels=frozenset({"ci-patchwork"})), - ) == Decimal("999.000") + ) == Decimal("-999.000") assert calculate_priority( entry, POLICY, PriorityContext(labels=frozenset({"ci-patchwork", "ci-patchwork-waived"})), - ) < Decimal("999.000") + ) > Decimal("-999.000") def test_maintainer_override_wins_and_conflicts_are_rejected(): @@ -93,7 +102,12 @@ def test_maintainer_override_wins_and_conflicts_are_rejected(): entry, POLICY, PriorityContext(labels=frozenset({"ci-priority:p0"})), - ) == Decimal("0.100") + ) == Decimal("0.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"ci-priority:p1000000"})), + ) == Decimal("1000000.000") with pytest.raises(ValueError, match="Multiple ci-priority override"): calculate_priority( entry, @@ -120,7 +134,7 @@ def test_annotation_only_touches_runnable_matrix_entries(): annotated = annotate_jobs(payload, POLICY) - assert annotated["single_node"]["1k1k"][0]["priority"] == "2.250" + assert annotated["single_node"]["1k1k"][0]["priority"] == "2.750" assert len(annotated["single_node"]["1k1k"][0]["queue-token"]) == 20 assert "priority" not in annotated["changelog_metadata"] assert "priority" not in payload["single_node"]["1k1k"][0] diff --git a/utils/test_ci_priority_controller.py b/utils/test_ci_priority_controller.py index 3f21010279..dcf42be61c 100644 --- a/utils/test_ci_priority_controller.py +++ b/utils/test_ci_priority_controller.py @@ -47,18 +47,18 @@ def test_assigns_best_compatible_job_to_each_idle_runner(): assert [(update.runner_name, update.assigned_job_id) for update in updates] == [ ("b200_00", 2), - ("h100_00", 3), + ("h100_00", 1), ] assert "ci-priority-p0.700" in updates[0].labels - assert "ci-priority-p2.500" in updates[1].labels + assert "ci-priority-p5.000" in updates[1].labels assert "ci-queue-job-2" in updates[0].labels - assert "ci-queue-job-3" in updates[1].labels + assert "ci-queue-job-1" in updates[1].labels def test_aging_breaks_starvation_between_nearby_priorities(): jobs = [ - job(1, "3.000", "h100", queued_minutes=8 * 60), - job(2, "1.500", "h100", queued_minutes=1), + job(1, "1.000", "h100", queued_minutes=8 * 60), + job(2, "2.500", "h100", queued_minutes=1), ] runners = [runner(11, "h100_00", "h100")] @@ -72,6 +72,19 @@ def test_aging_breaks_starvation_between_nearby_priorities(): assert updates[0].assigned_job_id == 1 +def test_core_skip_queue_outranks_any_numeric_priority(): + jobs = [ + job(1, "1000000.000", "h100"), + job(2, "skip", "h100"), + ] + runners = [runner(11, "h100_00", "h100")] + + updates = plan_label_updates(jobs, runners, now=NOW) + + assert updates[0].assigned_job_id == 2 + assert "ci-priority-pskip" in updates[0].labels + + def test_unused_idle_runner_loses_stale_priority_label(): runners = [ runner( From 164eaed04d5d9bb03232da0b50fcfbf489aaf8e9 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:03:25 -0500 Subject: [PATCH 04/21] Keep CI priorities nonnegative --- .../workflows/benchmark-multinode-tmpl.yml | 2 +- .github/workflows/benchmark-tmpl.yml | 2 +- .github/workflows/collectivex-sweep.yml | 2 +- .github/workflows/speedbench-al.yml | 2 +- configs/ci-priority.yaml | 8 +++---- utils/ci_priority_controller.py | 2 +- utils/test_ci_priority.py | 22 ++++++++++++------- 7 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 3bcb26eb82..6c85aaa3cd 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -10,7 +10,7 @@ on: description: "Higher-is-sooner CI priority score" required: false type: string - default: '0.000' + default: '1.000' queue-token: description: "One-shot queue token generated for this job" required: false diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4ce501d2b4..2c868cd9ae 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -9,7 +9,7 @@ on: description: "Higher-is-sooner CI priority score" required: false type: string - default: '0.000' + default: '1.000' queue-token: description: "One-shot queue token generated for this job" required: false diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml index 0351e4fd6b..7398decb6f 100644 --- a/.github/workflows/collectivex-sweep.yml +++ b/.github/workflows/collectivex-sweep.yml @@ -26,7 +26,7 @@ on: priority: description: Higher-is-sooner CI priority score type: string - default: '0.000' + default: '1.000' concurrency: group: cx-${{ github.ref }}-${{ inputs.backend }}-${{ inputs.only_sku }} cancel-in-progress: false diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index 968ca0816a..0e3931cf86 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -18,7 +18,7 @@ on: description: "Higher-is-sooner CI priority score" required: false type: string - default: '2.750' + default: '3.750' model: description: "HF model id (basename must be in launcher STAGED_MODELS for pre-staged local weights)" required: false diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index aba03d629e..97bc0f9035 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -2,14 +2,14 @@ version: 1 # Higher scores run first. Scores are uncapped and remain human-readable in # workflow job labels (for example, ci-priority-p2.750). -base-score: 0.0 +base-score: 1.0 adjustments: event: push: 2.0 multi-node: 1.25 agentic: 1.0 - eval-only: -0.5 + eval-only: 0.0 precision: fp4: 0.75 spec-decoding: @@ -33,7 +33,7 @@ adjustments: labels: # Repository labels are maintainer-controlled. A matching override wins over # every automatic adjustment. - override-pattern: '^ci-priority:p(?P-?[0-9]+(?:\.[0-9]+)?)$' + override-pattern: '^ci-priority:p(?P[0-9]+(?:\.[0-9]+)?)$' # pskip is reserved for an active skip_queue label whose latest labeling # actor is verified as an active SemiAnalysisAI/Core team member. skip-queue: @@ -46,7 +46,7 @@ labels: adjustment: 0.25 patchwork: names: [ci-patchwork, engine-patch] - score: -999.0 + score: 0.0 waived-by: [ci-patchwork-waived] scheduler: diff --git a/utils/ci_priority_controller.py b/utils/ci_priority_controller.py index 6fa84ee3ba..016224142c 100755 --- a/utils/ci_priority_controller.py +++ b/utils/ci_priority_controller.py @@ -21,7 +21,7 @@ import yaml PRIORITY_LABEL_RE = re.compile( - r"^ci-priority-p(?Pskip|-?[0-9]+(?:\.[0-9]+)?)$" + r"^ci-priority-p(?Pskip|[0-9]+(?:\.[0-9]+)?)$" ) QUEUE_LABEL_RE = re.compile(r"^ci-queue-(?P[a-zA-Z0-9._-]+)$") diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 18463086b5..8e8fe4470a 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -29,8 +29,8 @@ def test_combined_high_value_signals_outrank_baseline_job(): "decode": {"hardware": "b200"}, } - assert calculate_priority(high_value, POLICY) == Decimal("5.000") - assert calculate_priority(baseline, POLICY) == Decimal("0.000") + assert calculate_priority(high_value, POLICY) == Decimal("6.000") + assert calculate_priority(baseline, POLICY) == Decimal("1.000") def test_main_branch_jobs_receive_an_automatic_boost(): @@ -40,7 +40,7 @@ def test_main_branch_jobs_receive_an_automatic_boost(): entry, POLICY, PriorityContext(event_name="push"), - ) == Decimal("2.000") + ) == Decimal("3.000") def test_skip_queue_requires_both_label_and_core_authorization(): @@ -50,12 +50,12 @@ def test_skip_queue_requires_both_label_and_core_authorization(): entry, POLICY, PriorityContext(labels=frozenset({"skip_queue"})), - ) == Decimal("1.250") + ) == Decimal("2.250") assert calculate_priority( entry, POLICY, PriorityContext(skip_queue_authorized=True), - ) == Decimal("1.250") + ) == Decimal("2.250") assert calculate_priority( entry, POLICY, @@ -64,6 +64,7 @@ def test_skip_queue_requires_both_label_and_core_authorization(): skip_queue_authorized=True, ), ) == Decimal("Infinity") + annotated = annotate_jobs( [entry], POLICY, @@ -82,12 +83,12 @@ def test_patchwork_label_forces_bottom_priority_without_waiver(): entry, POLICY, PriorityContext(labels=frozenset({"ci-patchwork"})), - ) == Decimal("-999.000") + ) == Decimal("0.000") assert calculate_priority( entry, POLICY, PriorityContext(labels=frozenset({"ci-patchwork", "ci-patchwork-waived"})), - ) > Decimal("-999.000") + ) > Decimal("0.000") def test_maintainer_override_wins_and_conflicts_are_rejected(): @@ -108,6 +109,11 @@ def test_maintainer_override_wins_and_conflicts_are_rejected(): POLICY, PriorityContext(labels=frozenset({"ci-priority:p1000000"})), ) == Decimal("1000000.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"ci-priority:p-1"})), + ) == Decimal("1.000") with pytest.raises(ValueError, match="Multiple ci-priority override"): calculate_priority( entry, @@ -134,7 +140,7 @@ def test_annotation_only_touches_runnable_matrix_entries(): annotated = annotate_jobs(payload, POLICY) - assert annotated["single_node"]["1k1k"][0]["priority"] == "2.750" + assert annotated["single_node"]["1k1k"][0]["priority"] == "3.750" assert len(annotated["single_node"]["1k1k"][0]["queue-token"]) == 20 assert "priority" not in annotated["changelog_metadata"] assert "priority" not in payload["single_node"]["1k1k"][0] From 812a9416f3e32fa9c1cee2ccb6433da960a17a36 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:04:32 -0500 Subject: [PATCH 05/21] Reuse REPO_PAT for skip queue authorization --- .github/workflows/run-sweep.yml | 4 +++- utils/authorize_skip_queue.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 2a5fd7d677..82cced251f 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -222,8 +222,10 @@ jobs: if: >- github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'skip_queue') + # REPO_PAT needs repository Issues: Read. + # REPO_PAT needs organization Members: Read. env: - CORE_TEAM_READ_TOKEN: ${{ secrets.CORE_TEAM_READ_TOKEN }} + REPO_PAT: ${{ secrets.REPO_PAT }} run: | authorized=$(python3 "${GITHUB_WORKSPACE}/utils/authorize_skip_queue.py" \ --repository "${{ github.repository }}" \ diff --git a/utils/authorize_skip_queue.py b/utils/authorize_skip_queue.py index c509457d2d..4b262424d9 100755 --- a/utils/authorize_skip_queue.py +++ b/utils/authorize_skip_queue.py @@ -85,7 +85,7 @@ def main() -> int: parser.add_argument("--organization", default="SemiAnalysisAI") parser.add_argument("--team-slug", default="core") parser.add_argument("--label", default="skip_queue") - parser.add_argument("--token-env", default="CORE_TEAM_READ_TOKEN") + parser.add_argument("--token-env", default="REPO_PAT") parser.add_argument("--api-url", default="https://api.github.com") args = parser.parse_args() From 354f4ada34bdd4a36fb2300f7bac4778ce8025ca Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:29:52 -0500 Subject: [PATCH 06/21] Make CI priorities fully automatic --- .../workflows/benchmark-multinode-tmpl.yml | 29 ++- .github/workflows/benchmark-tmpl.yml | 29 ++- .github/workflows/collectivex-sweep.yml | 14 +- .github/workflows/run-sweep.yml | 29 +-- .github/workflows/speedbench-al.yml | 35 ++- configs/ci-priority.yaml | 7 +- utils/ci_priority.py | 39 +--- utils/ci_priority_controller.py | 207 ++++++++++++++---- utils/test_ci_priority.py | 62 ++---- utils/test_ci_priority_controller.py | 131 ++++++++++- 10 files changed, 401 insertions(+), 181 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 6c85aaa3cd..00c85a52df 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -8,14 +8,17 @@ on: type: string priority: description: "Higher-is-sooner CI priority score" - required: false + required: true type: string - default: '1.000' queue-token: description: "One-shot queue token generated for this job" + required: true + type: string + skip-queue-pr: + description: "Requested skip_queue PR number" required: false type: string - default: 'legacy' + default: '' image: required: true type: string @@ -244,11 +247,21 @@ jobs: runs-on: >- ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && - format( - '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', - inputs.runner, - inputs.priority, - inputs.queue-token + ( + inputs.skip-queue-pr != '' && + format( + '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}","ci-skip-queue-pr-{3}"]', + inputs.runner, + inputs.priority, + inputs.queue-token, + inputs.skip-queue-pr + ) || + format( + '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + inputs.runner, + inputs.priority, + inputs.queue-token + ) ) || format('["{0}"]', inputs.runner) ) }} diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 2c868cd9ae..4e5e81f4f4 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -7,14 +7,17 @@ on: type: string priority: description: "Higher-is-sooner CI priority score" - required: false + required: true type: string - default: '1.000' queue-token: description: "One-shot queue token generated for this job" + required: true + type: string + skip-queue-pr: + description: "Requested skip_queue PR number" required: false type: string - default: 'legacy' + default: '' image: required: true type: string @@ -174,11 +177,21 @@ jobs: runs-on: >- ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && - format( - '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', - inputs.runner, - inputs.priority, - inputs.queue-token + ( + inputs.skip-queue-pr != '' && + format( + '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}","ci-skip-queue-pr-{3}"]', + inputs.runner, + inputs.priority, + inputs.queue-token, + inputs.skip-queue-pr + ) || + format( + '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + inputs.runner, + inputs.priority, + inputs.queue-token + ) ) || format('["{0}"]', inputs.runner) ) }} diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml index 7398decb6f..eb7ddebd1d 100644 --- a/.github/workflows/collectivex-sweep.yml +++ b/.github/workflows/collectivex-sweep.yml @@ -23,10 +23,6 @@ on: description: Keep only shards whose expert-parallel degree is in this comma-list (8 keeps EP8 only, dropping EP16 so GB SKUs co-schedule with 8-GPU SKUs; blank = all) type: string default: '' - priority: - description: Higher-is-sooner CI priority score - type: string - default: '1.000' concurrency: group: cx-${{ github.ref }}-${{ inputs.backend }}-${{ inputs.only_sku }} cancel-in-progress: false @@ -38,9 +34,15 @@ jobs: outputs: matrix: ${{ steps.gen.outputs.matrix }} n: ${{ steps.gen.outputs.n }} + priority: ${{ steps.score.outputs.priority }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0 with: { clean: true, persist-credentials: false } + - id: score + run: | + scored=$(printf '%s' '[{"runner":"collectivex","framework":"collectivex"}]' | + python3 utils/ci_priority.py) + echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" - id: gen working-directory: experimental/CollectiveX env: @@ -93,13 +95,13 @@ jobs: format( '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}-{3}"]', matrix.sku, - inputs.priority, + needs.setup.outputs.priority, github.run_id, matrix.id ) || format('["{0}"]', matrix.sku) ) }} - name: p${{ inputs.priority }} | ${{ matrix.sku }} ${{ matrix.backend }} shard ${{ matrix.id }} + name: p${{ needs.setup.outputs.priority }} | ${{ matrix.sku }} ${{ matrix.backend }} shard ${{ matrix.id }} timeout-minutes: 350 env: COLLX_BENCH: ${{ matrix.backend }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 82cced251f..8642837c47 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -217,26 +217,10 @@ jobs: with: fetch-depth: 0 - - name: Authorize skip_queue - id: authorize_skip_queue - if: >- - github.event_name == 'pull_request' && - contains(github.event.pull_request.labels.*.name, 'skip_queue') - # REPO_PAT needs repository Issues: Read. - # REPO_PAT needs organization Members: Read. - env: - REPO_PAT: ${{ secrets.REPO_PAT }} - run: | - authorized=$(python3 "${GITHUB_WORKSPACE}/utils/authorize_skip_queue.py" \ - --repository "${{ github.repository }}" \ - --pr-number "${{ github.event.pull_request.number }}") - echo "authorized=$authorized" >> "$GITHUB_OUTPUT" - - id: setup env: GH_TOKEN: ${{ github.token }} PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} - SKIP_QUEUE_AUTHORIZED: ${{ steps.authorize_skip_queue.outputs.authorized == 'true' }} TRIM_CONC: >- ${{ github.event_name == 'pull_request' && @@ -279,10 +263,6 @@ jobs: CMD+=(--evals-only) fi - PRIORITY_ARGS=() - if [ "$SKIP_QUEUE_AUTHORIZED" = "true" ]; then - PRIORITY_ARGS+=(--skip-queue-authorized) - fi CONFIG_JSON=$("${CMD[@]}") CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ @@ -290,7 +270,7 @@ jobs: --event-name "${{ github.event_name }}" \ --queue-namespace "${{ github.run_id }}" \ --labels-json "$PR_LABELS" \ - "${PRIORITY_ARGS[@]}") + --pr-number "${{ github.event.pull_request.number || 0 }}") echo "search-space-config=$CONFIG_JSON" >> "$GITHUB_OUTPUT" python3 "${GITHUB_WORKSPACE}/utils/find_reusable_sweep_run.py" \ @@ -363,6 +343,7 @@ jobs: runner: ${{ matrix.config.runner }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -405,6 +386,7 @@ jobs: runner: ${{ matrix.config.runner }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -483,6 +465,7 @@ jobs: runner: ${{ matrix.config.runner }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -543,6 +526,7 @@ jobs: runner: ${{ matrix.config.runner }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -595,6 +579,7 @@ jobs: runner: ${{ matrix.config.runner }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -657,6 +642,7 @@ jobs: runner: ${{ matrix.config.runner }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} @@ -702,6 +688,7 @@ jobs: runner: ${{ matrix.config.runner }} priority: ${{ matrix.config.priority }} queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index 0e3931cf86..76968976c2 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -14,11 +14,6 @@ on: required: false type: string default: 'b300' - priority: - description: "Higher-is-sooner CI priority score" - required: false - type: string - default: '3.750' model: description: "HF model id (basename must be in launcher STAGED_MODELS for pre-staged local weights)" required: false @@ -108,20 +103,46 @@ env: PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache jobs: + setup: + runs-on: ubuntu-latest + outputs: + priority: ${{ steps.score.outputs.priority }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: { clean: true, persist-credentials: false } + - id: score + env: + RUNNER: ${{ inputs.runner }} + MODEL_PREFIX: ${{ inputs.model-prefix }} + run: | + entry=$(jq -cn \ + --arg runner "$RUNNER" \ + --arg model_prefix "$MODEL_PREFIX" \ + '[{ + runner: $runner, + framework: "vllm", + "model-prefix": $model_prefix, + precision: "fp4", + "spec-decoding": "mtp" + }]') + scored=$(printf '%s' "$entry" | python3 utils/ci_priority.py) + echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" + collect-al: + needs: setup runs-on: >- ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', inputs.runner, - inputs.priority, + needs.setup.outputs.priority, github.run_id ) || format('["{0}"]', inputs.runner) ) }} timeout-minutes: 600 - name: "p${{ inputs.priority }} | SpeedBench AL matrix | ${{ inputs.category }} | mtp=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" + name: "p${{ needs.setup.outputs.priority }} | SpeedBench AL matrix | ${{ inputs.category }} | mtp=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" steps: - name: Resource cleanup (pre-run) run: &resource-cleanup | diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index 97bc0f9035..bfd94b11bb 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -31,14 +31,9 @@ adjustments: dsr1: 0.75 labels: - # Repository labels are maintainer-controlled. A matching override wins over - # every automatic adjustment. - override-pattern: '^ci-priority:p(?P[0-9]+(?:\.[0-9]+)?)$' - # pskip is reserved for an active skip_queue label whose latest labeling - # actor is verified as an active SemiAnalysisAI/Core team member. + # Controller verifies skip_queue labeling actor membership. skip-queue: name: skip_queue - score: skip organization: SemiAnalysisAI team-slug: core checklist-complete: diff --git a/utils/ci_priority.py b/utils/ci_priority.py index d86ae4e601..ff0e11c96b 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -6,12 +6,11 @@ import argparse import hashlib import json -import re import sys from dataclasses import dataclass from decimal import Decimal, ROUND_HALF_UP from pathlib import Path -from typing import Any, Iterable +from typing import Any import yaml @@ -23,7 +22,7 @@ class PriorityContext: event_name: str = "" labels: frozenset[str] = frozenset() queue_namespace: str = "local" - skip_queue_authorized: bool = False + pr_number: int | None = None def _decimal(value: Any) -> Decimal: @@ -45,17 +44,6 @@ def _first_prefix_adjustment(value: str, adjustments: dict[str, Any]) -> Decimal return Decimal(0) -def _label_override(labels: Iterable[str], policy: dict[str, Any]) -> Decimal | None: - pattern = re.compile(policy["labels"]["override-pattern"]) - matches = [] - for label in labels: - match = pattern.fullmatch(label) - if match: - matches.append(_decimal(match.group("score"))) - if len(matches) > 1: - raise ValueError("Multiple ci-priority override labels are not allowed") - return matches[0] if matches else None - def calculate_priority( entry: dict[str, Any], @@ -63,18 +51,6 @@ def calculate_priority( context: PriorityContext = PriorityContext(), ) -> Decimal: """Return a higher-is-sooner priority score for one benchmark matrix entry.""" - override = _label_override(context.labels, policy) - skip_queue = policy["labels"]["skip-queue"] - if ( - context.skip_queue_authorized - and skip_queue["name"] in context.labels - ): - if skip_queue["score"] != "skip": - raise ValueError("Authorized skip_queue priority must use the 'skip' sentinel") - return Decimal("Infinity") - if override is not None: - return override.quantize(SCORE_QUANTUM) - patchwork = policy["labels"]["patchwork"] patch_labels = set(patchwork["names"]) waiver_labels = set(patchwork.get("waived-by", [])) @@ -111,7 +87,7 @@ def calculate_priority( def format_priority(score: Decimal) -> str: - return "skip" if score.is_infinite() else f"{score:.3f}" + return f"{score:.3f}" def queue_token(value: dict[str, Any], namespace: str, path: tuple[str, ...]) -> str: @@ -143,6 +119,11 @@ def annotate_jobs( if "runner" in value and "framework" in value: annotated["priority"] = format_priority(calculate_priority(value, policy, context)) annotated["queue-token"] = queue_token(value, context.queue_namespace, _path) + if ( + context.pr_number is not None + and policy["labels"]["skip-queue"]["name"] in context.labels + ): + annotated["skip-queue-pr"] = context.pr_number return annotated @@ -163,7 +144,7 @@ def main() -> int: parser.add_argument("--event-name", default="") parser.add_argument("--labels-json", default="[]") parser.add_argument("--queue-namespace", default="local") - parser.add_argument("--skip-queue-authorized", action="store_true") + parser.add_argument("--pr-number", type=int) parser.add_argument( "--input", type=Path, @@ -176,7 +157,7 @@ def main() -> int: event_name=args.event_name, labels=_labels_from_json(args.labels_json), queue_namespace=args.queue_namespace, - skip_queue_authorized=args.skip_queue_authorized, + pr_number=args.pr_number, ) source = args.input.read_text() if args.input else sys.stdin.read() payload = json.loads(source) diff --git a/utils/ci_priority_controller.py b/utils/ci_priority_controller.py index 016224142c..01fd6435a2 100755 --- a/utils/ci_priority_controller.py +++ b/utils/ci_priority_controller.py @@ -20,10 +20,16 @@ import yaml +from authorize_skip_queue import is_authorized + +PRIORITY_LABEL_PREFIX = "ci-priority-" +QUEUE_LABEL_PREFIX = "ci-queue-" +SKIP_QUEUE_LABEL_PREFIX = "ci-skip-queue-pr-" PRIORITY_LABEL_RE = re.compile( - r"^ci-priority-p(?Pskip|[0-9]+(?:\.[0-9]+)?)$" + r"^ci-priority-p(?P[0-9]+(?:\.[0-9]+)?)$" ) QUEUE_LABEL_RE = re.compile(r"^ci-queue-(?P[a-zA-Z0-9._-]+)$") +SKIP_QUEUE_LABEL_RE = re.compile(r"^ci-skip-queue-pr-(?P[1-9][0-9]*)$") def parse_timestamp(value: str | None) -> datetime: @@ -33,30 +39,49 @@ def parse_timestamp(value: str | None) -> datetime: def priority_from_labels(labels: Iterable[str]) -> tuple[str, Decimal] | None: - matches = [] - for label in labels: - match = PRIORITY_LABEL_RE.fullmatch(label) - if match: - raw_score = match.group("score") - score = Decimal("Infinity") if raw_score == "skip" else Decimal(raw_score) - matches.append((label, score)) - if len(matches) > 1: - raise ValueError(f"Job has multiple CI priority labels: {[label for label, _ in matches]}") - return matches[0] if matches else None + candidates = [label for label in labels if label.startswith(PRIORITY_LABEL_PREFIX)] + invalid = [label for label in candidates if not PRIORITY_LABEL_RE.fullmatch(label)] + if invalid: + raise ValueError(f"Job has invalid CI priority labels: {invalid}") + if len(candidates) > 1: + raise ValueError(f"Job has multiple CI priority labels: {candidates}") + if not candidates: + return None + label = candidates[0] + match = PRIORITY_LABEL_RE.fullmatch(label) + return label, Decimal(match.group("score")) def queue_label_from_labels(labels: Iterable[str]) -> str | None: - matches = [label for label in labels if QUEUE_LABEL_RE.fullmatch(label)] - if len(matches) > 1: - raise ValueError(f"Job has multiple CI queue labels: {matches}") - return matches[0] if matches else None + candidates = [label for label in labels if label.startswith(QUEUE_LABEL_PREFIX)] + invalid = [label for label in candidates if not QUEUE_LABEL_RE.fullmatch(label)] + if invalid: + raise ValueError(f"Job has invalid CI queue labels: {invalid}") + if len(candidates) > 1: + raise ValueError(f"Job has multiple CI queue labels: {candidates}") + return candidates[0] if candidates else None + +def skip_queue_from_labels(labels: Iterable[str]) -> tuple[str, int] | None: + candidates = [label for label in labels if label.startswith(SKIP_QUEUE_LABEL_PREFIX)] + invalid = [label for label in candidates if not SKIP_QUEUE_LABEL_RE.fullmatch(label)] + if invalid: + raise ValueError(f"Job has invalid skip_queue labels: {invalid}") + if len(candidates) > 1: + raise ValueError(f"Job has multiple skip_queue labels: {candidates}") + if not candidates: + return None + label = candidates[0] + match = SKIP_QUEUE_LABEL_RE.fullmatch(label) + return label, int(match.group("number")) def without_scheduling_labels(labels: Iterable[str]) -> frozenset[str]: return frozenset( label for label in labels - if not PRIORITY_LABEL_RE.fullmatch(label) and not QUEUE_LABEL_RE.fullmatch(label) + if not label.startswith( + (PRIORITY_LABEL_PREFIX, QUEUE_LABEL_PREFIX, SKIP_QUEUE_LABEL_PREFIX) + ) ) @@ -121,20 +146,31 @@ def effective_priority( *, now: datetime, aging_per_hour: Decimal, + authorized_skip_prs: frozenset[int] = frozenset(), ) -> Decimal: priority = priority_from_labels(job.labels) if priority is None: raise ValueError(f"Job {job.id} has no CI priority label") + skip_request = skip_queue_from_labels(job.labels) + if skip_request is not None and skip_request[1] in authorized_skip_prs: + return Decimal("Infinity") _, score = priority waited_hours = Decimal(str(max(0.0, (now - job.queued_at).total_seconds()) / 3600)) return score + waited_hours * aging_per_hour +def is_compatible(job: QueuedJob, runner: Runner) -> bool: + required = without_scheduling_labels(job.labels) + available = without_scheduling_labels(runner.labels) + return required.issubset(available) + + def plan_label_updates( jobs: Iterable[QueuedJob], runners: Iterable[Runner], *, aging_per_hour: Decimal = Decimal("0.25"), + authorized_skip_prs: frozenset[int] = frozenset(), now: datetime | None = None, ) -> list[LabelUpdate]: """Assign each idle runner to the best compatible queued job. @@ -144,16 +180,21 @@ def plan_label_updates( second job before the controller has considered newly queued higher priorities. """ - now = now or datetime.now(timezone.utc) - eligible_jobs = [ - job - for job in jobs - if priority_from_labels(job.labels) is not None - and queue_label_from_labels(job.labels) is not None - ] + eligible_jobs = [] + for job in jobs: + priority = priority_from_labels(job.labels) + queue_label = queue_label_from_labels(job.labels) + skip_queue_from_labels(job.labels) + if priority is not None and queue_label is not None: + eligible_jobs.append(job) eligible_jobs.sort( key=lambda job: ( - -effective_priority(job, now=now, aging_per_hour=aging_per_hour), + -effective_priority( + job, + now=now, + aging_per_hour=aging_per_hour, + authorized_skip_prs=authorized_skip_prs, + ), job.queued_at, job.id, ) @@ -166,23 +207,31 @@ def plan_label_updates( for runner in idle_runners } - for job in eligible_jobs: + for index, job in enumerate(eligible_jobs): priority = priority_from_labels(job.labels) queue_label = queue_label_from_labels(job.labels) + skip_request = skip_queue_from_labels(job.labels) if priority is None or queue_label is None: continue priority_label, _ = priority - required = without_scheduling_labels(job.labels) candidates = [ - runner - for runner in remaining.values() - if required.issubset(without_scheduling_labels(runner.labels)) + runner for runner in remaining.values() if is_compatible(job, runner) ] if not candidates: continue - runner = min(candidates, key=lambda candidate: (candidate.name, candidate.id)) + later_jobs = eligible_jobs[index + 1:] + runner = min( + candidates, + key=lambda candidate: ( + sum(is_compatible(later, candidate) for later in later_jobs), + candidate.name, + candidate.id, + ), + ) desired[runner.id] = ( - without_scheduling_labels(runner.labels) | {priority_label, queue_label}, + without_scheduling_labels(runner.labels) + | {priority_label, queue_label} + | ({skip_request[0]} if skip_request is not None else set()), job.id, ) del remaining[runner.id] @@ -229,15 +278,21 @@ def request(self, method: str, path: str, body: Any = None) -> Any: except urllib.error.HTTPError as error: detail = error.read().decode(errors="replace") raise RuntimeError(f"GitHub API {method} {path} failed: {error.code} {detail}") from error + except urllib.error.URLError as error: + raise RuntimeError(f"GitHub API {method} {path} failed: {error}") from error return json.loads(raw) if raw else None - def paged(self, path: str, key: str) -> list[dict[str, Any]]: + def paged( + self, + path: str, + key: str | None, + ) -> list[dict[str, Any]]: separator = "&" if "?" in path else "?" values = [] page = 1 while True: payload = self.request("GET", f"{path}{separator}per_page=100&page={page}") - batch = payload[key] + batch = payload if key is None else payload[key] values.extend(batch) if len(batch) < 100: return values @@ -278,17 +333,69 @@ def replace_runner_labels(self, runner_id: int, labels: Iterable[str]) -> None: ) -def load_aging_rate(policy_path: str | Path) -> Decimal: +class AuthorizationApi: + def __init__(self, client: GitHubClient): + self.client = client + + def paged(self, path: str) -> list[dict[str, Any]]: + return self.client.paged(path, None) + + def request(self, path: str) -> Any: + return self.client.request("GET", path) + + +def load_policy(policy_path: str | Path) -> dict[str, Any]: with Path(policy_path).open() as policy_file: - policy = yaml.safe_load(policy_file) + return yaml.safe_load(policy_file) + + +def authorize_skip_requests( + client: GitHubClient, + jobs: Iterable[QueuedJob], + policy: dict[str, Any], +) -> frozenset[int]: + requests = { + request[1] + for job in jobs + if (request := skip_queue_from_labels(job.labels)) is not None + } + skip_policy = policy["labels"]["skip-queue"] + api = AuthorizationApi(client) + authorized = set() + for pr_number in sorted(requests): + try: + allowed, actor = is_authorized( + api, + repository=client.repository, + pr_number=pr_number, + organization=skip_policy["organization"], + team_slug=skip_policy["team-slug"], + label_name=skip_policy["name"], + ) + except RuntimeError as error: + print(f"::warning::{error}; refusing skip_queue authorization", file=sys.stderr) + continue + if allowed: + print(f"::notice::skip_queue authorized by {actor}", file=sys.stderr) + authorized.add(pr_number) + return frozenset(authorized) + + +def load_aging_rate(policy: dict[str, Any]) -> Decimal: return Decimal(str(policy["scheduler"]["aging-per-hour"])) -def reconcile(client: GitHubClient, aging_per_hour: Decimal, apply: bool) -> list[LabelUpdate]: +def reconcile( + client: GitHubClient, + policy: dict[str, Any], + apply: bool, +) -> list[LabelUpdate]: + jobs = client.queued_jobs() updates = plan_label_updates( - client.queued_jobs(), + jobs, client.runners(), - aging_per_hour=aging_per_hour, + aging_per_hour=load_aging_rate(policy), + authorized_skip_prs=authorize_skip_requests(client, jobs, policy), ) if apply: for update in updates: @@ -310,16 +417,19 @@ def main() -> int: plan_parser.add_argument("--jobs", type=Path, required=True) plan_parser.add_argument("--runners", type=Path, required=True) plan_parser.add_argument("--now", help="ISO-8601 clock for deterministic simulations") + plan_parser.add_argument("--authorized-skip-pr", action="append", type=int, default=[]) reconcile_parser = subparsers.add_parser("reconcile", help="Poll GitHub and reconcile runner labels") reconcile_parser.add_argument("--repository", required=True) reconcile_parser.add_argument("--api-url", default="https://api.github.com") - reconcile_parser.add_argument("--token-env", default="GITHUB_TOKEN") + # REPO_PAT needs Actions and Issues read. + # REPO_PAT needs Administration write and Members read. + reconcile_parser.add_argument("--token-env", default="REPO_PAT") reconcile_parser.add_argument("--apply", action="store_true") reconcile_parser.add_argument("--watch", type=float, default=0, metavar="SECONDS") args = parser.parse_args() - aging_per_hour = load_aging_rate(args.policy) + policy = load_policy(args.policy) if args.command == "plan": raw_jobs = _load_json(args.jobs) @@ -327,7 +437,13 @@ def main() -> int: jobs = [QueuedJob.from_payload(job) for job in raw_jobs] runners = [Runner.from_payload(runner) for runner in raw_runners] now = parse_timestamp(args.now) if args.now else None - updates = plan_label_updates(jobs, runners, aging_per_hour=aging_per_hour, now=now) + updates = plan_label_updates( + jobs, + runners, + aging_per_hour=load_aging_rate(policy), + authorized_skip_prs=frozenset(args.authorized_skip_pr), + now=now, + ) json.dump([update.as_dict() for update in updates], sys.stdout, indent=2) sys.stdout.write("\n") return 0 @@ -337,7 +453,14 @@ def main() -> int: parser.error(f"{args.token_env} must contain a GitHub token") client = GitHubClient(args.repository, token, args.api_url) while True: - updates = reconcile(client, aging_per_hour, args.apply) + try: + updates = reconcile(client, policy, args.apply) + except RuntimeError as error: + if args.watch <= 0: + raise + print(f"::warning::{error}; retrying", file=sys.stderr) + time.sleep(args.watch) + continue print(json.dumps([update.as_dict() for update in updates], separators=(",", ":"))) if args.watch <= 0: return 0 diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 8e8fe4470a..c8c2700f3c 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -1,8 +1,6 @@ from decimal import Decimal from pathlib import Path -import pytest - from ci_priority import PriorityContext, annotate_jobs, calculate_priority, load_policy @@ -43,37 +41,25 @@ def test_main_branch_jobs_receive_an_automatic_boost(): ) == Decimal("3.000") -def test_skip_queue_requires_both_label_and_core_authorization(): +def test_skip_queue_request_keeps_numeric_priority(): entry = {"runner": "h100", "framework": "sglang", "precision": "fp4"} - assert calculate_priority( - entry, - POLICY, - PriorityContext(labels=frozenset({"skip_queue"})), - ) == Decimal("2.250") - assert calculate_priority( - entry, - POLICY, - PriorityContext(skip_queue_authorized=True), - ) == Decimal("2.250") - assert calculate_priority( - entry, - POLICY, - PriorityContext( - labels=frozenset({"skip_queue"}), - skip_queue_authorized=True, - ), - ) == Decimal("Infinity") - annotated = annotate_jobs( [entry], POLICY, PriorityContext( labels=frozenset({"skip_queue"}), - skip_queue_authorized=True, + pr_number=2124, ), ) - assert annotated[0]["priority"] == "skip" + + assert calculate_priority( + entry, + POLICY, + PriorityContext(labels=frozenset({"skip_queue"})), + ) == Decimal("2.250") + assert annotated[0]["priority"] == "2.250" + assert annotated[0]["skip-queue-pr"] == 2124 def test_patchwork_label_forces_bottom_priority_without_waiver(): @@ -91,35 +77,17 @@ def test_patchwork_label_forces_bottom_priority_without_waiver(): ) > Decimal("0.000") -def test_maintainer_override_wins_and_conflicts_are_rejected(): +def test_priority_labels_do_not_override_automatic_score(): entry = {"runner": "h100", "framework": "trt"} + labels = frozenset( + {"ci-priority:p0", "ci-priority:p4.5", "ci-priority:p1000000"} + ) assert calculate_priority( entry, POLICY, - PriorityContext(labels=frozenset({"ci-priority:p0.7"})), - ) == Decimal("0.700") - assert calculate_priority( - entry, - POLICY, - PriorityContext(labels=frozenset({"ci-priority:p0"})), - ) == Decimal("0.000") - assert calculate_priority( - entry, - POLICY, - PriorityContext(labels=frozenset({"ci-priority:p1000000"})), - ) == Decimal("1000000.000") - assert calculate_priority( - entry, - POLICY, - PriorityContext(labels=frozenset({"ci-priority:p-1"})), + PriorityContext(labels=labels), ) == Decimal("1.000") - with pytest.raises(ValueError, match="Multiple ci-priority override"): - calculate_priority( - entry, - POLICY, - PriorityContext(labels=frozenset({"ci-priority:p0.7", "ci-priority:p2.5"})), - ) def test_annotation_only_touches_runnable_matrix_entries(): diff --git a/utils/test_ci_priority_controller.py b/utils/test_ci_priority_controller.py index dcf42be61c..49df6450f6 100644 --- a/utils/test_ci_priority_controller.py +++ b/utils/test_ci_priority_controller.py @@ -1,7 +1,16 @@ +from dataclasses import replace from datetime import datetime, timedelta, timezone from decimal import Decimal -from ci_priority_controller import GitHubClient, QueuedJob, Runner, plan_label_updates +import pytest + +from ci_priority_controller import ( + GitHubClient, + QueuedJob, + Runner, + authorize_skip_requests, + plan_label_updates, +) NOW = datetime(2026, 7, 14, 18, 0, tzinfo=timezone.utc) @@ -68,21 +77,88 @@ def test_aging_breaks_starvation_between_nearby_priorities(): now=NOW, aging_per_hour=Decimal("0.25"), ) - assert updates[0].assigned_job_id == 1 - -def test_core_skip_queue_outranks_any_numeric_priority(): +def test_authorized_skip_queue_outranks_numeric_priority(): + skip_job = job(2, "1.000", "h100") + skip_job = replace( + skip_job, + labels=skip_job.labels | {"ci-skip-queue-pr-2124"}, + ) jobs = [ job(1, "1000000.000", "h100"), - job(2, "skip", "h100"), + skip_job, ] runners = [runner(11, "h100_00", "h100")] - updates = plan_label_updates(jobs, runners, now=NOW) + updates = plan_label_updates( + jobs, + runners, + authorized_skip_prs=frozenset({2124}), + now=NOW, + ) assert updates[0].assigned_job_id == 2 - assert "ci-priority-pskip" in updates[0].labels + assert "ci-priority-p1.000" in updates[0].labels + assert "ci-skip-queue-pr-2124" in updates[0].labels + + +def test_unauthorized_skip_queue_remains_numeric(): + skip_job = job(2, "1.000", "h100") + skip_job = replace( + skip_job, + labels=skip_job.labels | {"ci-skip-queue-pr-2124"}, + ) + + updates = plan_label_updates( + [job(1, "2.000", "h100"), skip_job], + [runner(11, "h100_00", "h100")], + now=NOW, + ) + + assert updates[0].assigned_job_id == 1 + + + +def test_controller_verifies_skip_queue_actor(): + class FixtureClient(GitHubClient): + def __init__(self): + super().__init__("owner/repo", "unused") + + def paged(self, path, key): + assert key is None + assert path == "/repos/owner/repo/issues/2124/timeline" + return [{ + "event": "labeled", + "label": {"name": "skip_queue"}, + "actor": {"login": "alice"}, + }] + + def request(self, method, path, body=None): + assert method == "GET" + assert path == "/orgs/SemiAnalysisAI/teams/core/memberships/alice" + return {"state": "active"} + + skip_job = job(2, "1.000", "h100") + skip_job = replace( + skip_job, + labels=skip_job.labels | {"ci-skip-queue-pr-2124"}, + ) + policy = { + "labels": { + "skip-queue": { + "name": "skip_queue", + "organization": "SemiAnalysisAI", + "team-slug": "core", + } + } + } + + assert authorize_skip_requests( + FixtureClient(), + [skip_job], + policy, + ) == frozenset({2124}) def test_unused_idle_runner_loses_stale_priority_label(): @@ -127,6 +203,47 @@ def test_exact_runner_name_remains_a_compatibility_constraint(): assert assigned[0].runner_name == "h100_01" +def test_preserves_scarce_runner_for_exact_job(): + jobs = [ + job(1, "5.000", "h100"), + job(2, "4.000", "h100_00"), + ] + runners = [ + runner(10, "h100_00", "h100"), + runner(11, "h100_01", "h100"), + ] + + updates = plan_label_updates(jobs, runners, now=NOW) + + assert [(update.runner_name, update.assigned_job_id) for update in updates] == [ + ("h100_00", 2), + ("h100_01", 1), + ] + + +@pytest.mark.parametrize( + ("valid", "invalid", "message"), + [ + ("ci-priority-p1.000", "ci-priority-p-1", "invalid CI priority"), + ("ci-queue-job-1", "ci-queue-", "invalid CI queue"), + ("ci-skip-queue-pr-2124", "ci-skip-queue-pr-zero", "invalid skip_queue"), + ], +) +def test_rejects_malformed_scheduling_labels(valid, invalid, message): + queued_job = job(1, "1.000", "h100") + queued_job = replace( + queued_job, + labels=(queued_job.labels - {valid}) | {invalid}, + ) + + with pytest.raises(ValueError, match=message): + plan_label_updates( + [queued_job], + [runner(11, "h100_00", "h100")], + now=NOW, + ) + + def test_discovers_queued_jobs_inside_in_progress_workflow_runs(): class FixtureClient(GitHubClient): def __init__(self): From 1243dcc561ab5080fcd0e2f2e6a1e3e10baaa01b Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:44:18 -0500 Subject: [PATCH 07/21] Address priority scheduler review findings --- utils/authorize_skip_queue.py | 32 ++++++++++++++++++++++------ utils/ci_priority.py | 2 +- utils/ci_priority_controller.py | 3 ++- utils/test_authorize_skip_queue.py | 16 +++++++++++++- utils/test_ci_priority.py | 13 +++++++++++ utils/test_ci_priority_controller.py | 14 ++++++++++++ 6 files changed, 70 insertions(+), 10 deletions(-) diff --git a/utils/authorize_skip_queue.py b/utils/authorize_skip_queue.py index 4b262424d9..415ed16a99 100755 --- a/utils/authorize_skip_queue.py +++ b/utils/authorize_skip_queue.py @@ -9,8 +9,11 @@ import sys import urllib.error import urllib.request +from pathlib import Path from typing import Any +import yaml + class GitHubApi: def __init__(self, token: str, api_url: str = "https://api.github.com"): @@ -78,16 +81,31 @@ def is_authorized( return membership.get("state") == "active", actor +def load_skip_policy(path: str | Path) -> dict[str, Any]: + with Path(path).open() as policy_file: + policy = yaml.safe_load(policy_file) + return policy["labels"]["skip-queue"] + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repository", required=True) parser.add_argument("--pr-number", required=True, type=int) - parser.add_argument("--organization", default="SemiAnalysisAI") - parser.add_argument("--team-slug", default="core") - parser.add_argument("--label", default="skip_queue") + parser.add_argument( + "--policy", + type=Path, + default=Path(__file__).parents[1] / "configs" / "ci-priority.yaml", + ) + parser.add_argument("--organization") + parser.add_argument("--team-slug") + parser.add_argument("--label") parser.add_argument("--token-env", default="REPO_PAT") parser.add_argument("--api-url", default="https://api.github.com") args = parser.parse_args() + skip_policy = load_skip_policy(args.policy) + organization = args.organization or skip_policy["organization"] + team_slug = args.team_slug or skip_policy["team-slug"] + label_name = args.label or skip_policy["name"] token = os.environ.get(args.token_env) if not token: @@ -100,9 +118,9 @@ def main() -> int: GitHubApi(token, args.api_url), repository=args.repository, pr_number=args.pr_number, - organization=args.organization, - team_slug=args.team_slug, - label_name=args.label, + organization=organization, + team_slug=team_slug, + label_name=label_name, ) except RuntimeError as error: print(f"::warning::{error}; refusing skip_queue authorization", file=sys.stderr) @@ -114,7 +132,7 @@ def main() -> int: elif actor: print( f"::warning::skip_queue was applied by {actor}, who is not an active " - f"{args.organization}/{args.team_slug} member", + f"{organization}/{team_slug} member", file=sys.stderr, ) print("true" if authorized else "false") diff --git a/utils/ci_priority.py b/utils/ci_priority.py index ff0e11c96b..c1ba379856 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -55,7 +55,7 @@ def calculate_priority( patch_labels = set(patchwork["names"]) waiver_labels = set(patchwork.get("waived-by", [])) if context.labels & patch_labels and not context.labels & waiver_labels: - return _decimal(patchwork["score"]).quantize(SCORE_QUANTUM) + return _decimal(patchwork["score"]).quantize(SCORE_QUANTUM, ROUND_HALF_UP) adjustments = policy["adjustments"] score = _decimal(policy["base-score"]) diff --git a/utils/ci_priority_controller.py b/utils/ci_priority_controller.py index 01fd6435a2..04508fca05 100755 --- a/utils/ci_priority_controller.py +++ b/utils/ci_priority_controller.py @@ -35,7 +35,8 @@ def parse_timestamp(value: str | None) -> datetime: if not value: return datetime.now(timezone.utc) - return datetime.fromisoformat(value.replace("Z", "+00:00")) + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) def priority_from_labels(labels: Iterable[str]) -> tuple[str, Decimal] | None: diff --git a/utils/test_authorize_skip_queue.py b/utils/test_authorize_skip_queue.py index f0f8c5066a..bc800bb401 100644 --- a/utils/test_authorize_skip_queue.py +++ b/utils/test_authorize_skip_queue.py @@ -1,4 +1,6 @@ -from authorize_skip_queue import active_label_actor, is_authorized +from pathlib import Path + +from authorize_skip_queue import active_label_actor, is_authorized, load_skip_policy class FakeApi: @@ -44,6 +46,18 @@ def authorize(api): ) +def test_policy_configures_skip_queue_authorization(): + policy = load_skip_policy( + Path(__file__).parents[1] / "configs" / "ci-priority.yaml" + ) + + assert policy == { + "name": "skip_queue", + "organization": "SemiAnalysisAI", + "team-slug": "core", + } + + def test_active_label_from_active_core_member_is_authorized(): api = FakeApi([labeled("alice")], {"alice": {"state": "active"}}) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index c8c2700f3c..6db35d6b90 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -1,3 +1,4 @@ +from copy import deepcopy from decimal import Decimal from pathlib import Path @@ -41,6 +42,18 @@ def test_main_branch_jobs_receive_an_automatic_boost(): ) == Decimal("3.000") +def test_patchwork_score_uses_half_up_rounding(): + policy = deepcopy(POLICY) + policy["labels"]["patchwork"]["score"] = 0.7225 + entry = {"runner": "h100", "framework": "trt"} + + assert calculate_priority( + entry, + policy, + PriorityContext(labels=frozenset({"ci-patchwork"})), + ) == Decimal("0.723") + + def test_skip_queue_request_keeps_numeric_priority(): entry = {"runner": "h100", "framework": "sglang", "precision": "fp4"} diff --git a/utils/test_ci_priority_controller.py b/utils/test_ci_priority_controller.py index 49df6450f6..5f38436637 100644 --- a/utils/test_ci_priority_controller.py +++ b/utils/test_ci_priority_controller.py @@ -9,6 +9,7 @@ QueuedJob, Runner, authorize_skip_requests, + parse_timestamp, plan_label_updates, ) @@ -41,6 +42,19 @@ def runner(runner_id, name, *labels, busy=False, status="online"): ) +def test_naive_timestamps_default_to_utc(): + now = parse_timestamp("2026-07-14T18:00:00") + + updates = plan_label_updates( + [job(1, "1.000", "h100")], + [runner(11, "h100_00", "h100")], + now=now, + ) + + assert now.tzinfo == timezone.utc + assert updates[0].assigned_job_id == 1 + + def test_assigns_best_compatible_job_to_each_idle_runner(): jobs = [ job(1, "5.000", "h100", queued_minutes=30), From f78651589827aef859df9a64e40a5acfe26ddfb3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:38:09 -0500 Subject: [PATCH 08/21] Drop priority label p prefix --- .../workflows/benchmark-multinode-tmpl.yml | 4 ++-- .github/workflows/benchmark-tmpl.yml | 4 ++-- .github/workflows/collectivex-sweep.yml | 2 +- .github/workflows/profile.yml | 2 +- .github/workflows/speedbench-al.yml | 2 +- configs/ci-priority.yaml | 2 +- utils/ci_priority_controller.py | 2 +- utils/test_ci_priority_controller.py | 20 +++++++++---------- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 00c85a52df..ee9241dc3e 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -250,14 +250,14 @@ jobs: ( inputs.skip-queue-pr != '' && format( - '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}","ci-skip-queue-pr-{3}"]', + '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}","ci-skip-queue-pr-{3}"]', inputs.runner, inputs.priority, inputs.queue-token, inputs.skip-queue-pr ) || format( - '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}"]', inputs.runner, inputs.priority, inputs.queue-token diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4e5e81f4f4..5451c8cc46 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -180,14 +180,14 @@ jobs: ( inputs.skip-queue-pr != '' && format( - '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}","ci-skip-queue-pr-{3}"]', + '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}","ci-skip-queue-pr-{3}"]', inputs.runner, inputs.priority, inputs.queue-token, inputs.skip-queue-pr ) || format( - '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}"]', inputs.runner, inputs.priority, inputs.queue-token diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml index eb7ddebd1d..e7994b7569 100644 --- a/.github/workflows/collectivex-sweep.yml +++ b/.github/workflows/collectivex-sweep.yml @@ -93,7 +93,7 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( - '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}-{3}"]', + '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}-{3}"]', matrix.sku, needs.setup.outputs.priority, github.run_id, diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index c352f42f58..a505b196bb 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -110,7 +110,7 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( - '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}"]', matrix.config.runner, matrix.config.priority, matrix.config['queue-token'] diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index 76968976c2..e5b5fa1f72 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -134,7 +134,7 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( - '["self-hosted","{0}","ci-priority-p{1}","ci-queue-{2}"]', + '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}"]', inputs.runner, needs.setup.outputs.priority, github.run_id diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index bfd94b11bb..10d3c1b290 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -1,7 +1,7 @@ version: 1 # Higher scores run first. Scores are uncapped and remain human-readable in -# workflow job labels (for example, ci-priority-p2.750). +# workflow job labels (for example, ci-priority-2.750). base-score: 1.0 adjustments: diff --git a/utils/ci_priority_controller.py b/utils/ci_priority_controller.py index 04508fca05..65c8c7a497 100755 --- a/utils/ci_priority_controller.py +++ b/utils/ci_priority_controller.py @@ -26,7 +26,7 @@ QUEUE_LABEL_PREFIX = "ci-queue-" SKIP_QUEUE_LABEL_PREFIX = "ci-skip-queue-pr-" PRIORITY_LABEL_RE = re.compile( - r"^ci-priority-p(?P[0-9]+(?:\.[0-9]+)?)$" + r"^ci-priority-(?P[0-9]+(?:\.[0-9]+)?)$" ) QUEUE_LABEL_RE = re.compile(r"^ci-queue-(?P[a-zA-Z0-9._-]+)$") SKIP_QUEUE_LABEL_RE = re.compile(r"^ci-skip-queue-pr-(?P[1-9][0-9]*)$") diff --git a/utils/test_ci_priority_controller.py b/utils/test_ci_priority_controller.py index 5f38436637..8f44c3ec0a 100644 --- a/utils/test_ci_priority_controller.py +++ b/utils/test_ci_priority_controller.py @@ -24,7 +24,7 @@ def job(job_id, priority, hardware, queued_minutes=0): labels=frozenset({ "self-hosted", hardware, - f"ci-priority-p{priority}", + f"ci-priority-{priority}", f"ci-queue-job-{job_id}", }), queued_at=NOW - timedelta(minutes=queued_minutes), @@ -72,8 +72,8 @@ def test_assigns_best_compatible_job_to_each_idle_runner(): ("b200_00", 2), ("h100_00", 1), ] - assert "ci-priority-p0.700" in updates[0].labels - assert "ci-priority-p5.000" in updates[1].labels + assert "ci-priority-0.700" in updates[0].labels + assert "ci-priority-5.000" in updates[1].labels assert "ci-queue-job-2" in updates[0].labels assert "ci-queue-job-1" in updates[1].labels @@ -113,7 +113,7 @@ def test_authorized_skip_queue_outranks_numeric_priority(): ) assert updates[0].assigned_job_id == 2 - assert "ci-priority-p1.000" in updates[0].labels + assert "ci-priority-1.000" in updates[0].labels assert "ci-skip-queue-pr-2124" in updates[0].labels @@ -181,7 +181,7 @@ def test_unused_idle_runner_loses_stale_priority_label(): 10, "b200_00", "b200", - "ci-priority-p5.000", + "ci-priority-5.000", "ci-queue-old-job", ) ] @@ -190,14 +190,14 @@ def test_unused_idle_runner_loses_stale_priority_label(): assert len(updates) == 1 assert updates[0].assigned_job_id is None - assert all(not label.startswith("ci-priority-p") for label in updates[0].labels) + assert all(not label.startswith("ci-priority-") for label in updates[0].labels) assert all(not label.startswith("ci-queue-") for label in updates[0].labels) def test_busy_and_offline_runners_are_never_relabelled(): runners = [ - runner(10, "b200_00", "b200", "ci-priority-p5.000", busy=True), - runner(11, "b200_01", "b200", "ci-priority-p5.000", status="offline"), + runner(10, "b200_00", "b200", "ci-priority-5.000", busy=True), + runner(11, "b200_01", "b200", "ci-priority-5.000", status="offline"), ] assert plan_label_updates([job(1, "0.700", "b200")], runners, now=NOW) == [] @@ -238,7 +238,7 @@ def test_preserves_scarce_runner_for_exact_job(): @pytest.mark.parametrize( ("valid", "invalid", "message"), [ - ("ci-priority-p1.000", "ci-priority-p-1", "invalid CI priority"), + ("ci-priority-1.000", "ci-priority--1", "invalid CI priority"), ("ci-queue-job-1", "ci-queue-", "invalid CI queue"), ("ci-skip-queue-pr-2124", "ci-skip-queue-pr-zero", "invalid skip_queue"), ], @@ -280,7 +280,7 @@ def paged(self, path, key): "labels": [ "self-hosted", "b200", - "ci-priority-p1.000", + "ci-priority-1.000", "ci-queue-job-22", ], }] From 9b28e05eda7c8f591f80dff7102146b9fe16dad1 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:17:04 -0500 Subject: [PATCH 09/21] Classify patchwork sweeps with Fable --- .github/workflows/run-sweep.yml | 80 +++++++++++++++++++++++++++++++-- utils/ci_priority.py | 8 +++- utils/test_ci_priority.py | 13 ++++++ 3 files changed, 97 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 8642837c47..ee94c299e8 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -154,8 +154,73 @@ jobs: --ref "${{ github.ref }}" \ --workflow-id "run-sweep.yml" + classify-patchwork: + needs: check-changelog + runs-on: ubuntu-latest + if: >- + always() && + github.event_name == 'pull_request' && + !github.event.pull_request.draft && + ( + needs.check-changelog.result == 'success' || + needs.check-changelog.result == 'skipped' + ) + permissions: + contents: read + pull-requests: read + outputs: + patchwork: ${{ steps.result.outputs.patchwork }} + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + + - name: Classify patchwork with Fable + id: fable + continue-on-error: true + uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + track_progress: false + settings: | + {"fastMode": true} + claude_args: | + --model 'claude-fable-5' + --max-turns 8 + --allowedTools "Read,Glob,Grep,Bash(git diff:*)" + --json-schema '{"type":"object","properties":{"patchwork":{"type":"boolean"},"reason":{"type":"string"}},"required":["patchwork","reason"]}' + prompt: | + Inspect this PR's diff against its base branch. + + Return patchwork=true only when the benchmark uses modified + upstream inference-engine or runtime source. This includes + patch files, git apply, source mutation, monkey-patching, + local engine builds, forks, or images containing unmerged + engine changes. + + Configuration and recipe changes using released or upstream + software are not patchwork. + + Return a concise reason grounded in the diff. + + - name: Normalize classification + id: result + if: always() + env: + OUTPUT: ${{ steps.fable.outputs.structured_output || '{}' }} + run: | + if jq -e 'type == "object" and (.patchwork | type == "boolean")' \ + >/dev/null <<<"$OUTPUT"; then + patchwork=$(jq -r '.patchwork' <<<"$OUTPUT") + else + patchwork=true + fi + echo "patchwork=$patchwork" >> "$GITHUB_OUTPUT" + echo "Fable patchwork classification: $patchwork" + setup: - needs: [check-changelog, reuse-sweep-gate] + needs: [check-changelog, reuse-sweep-gate, classify-patchwork] runs-on: ubuntu-latest permissions: actions: read @@ -175,6 +240,10 @@ jobs: needs.reuse-sweep-gate.outputs.skip-pr-sweep != 'true' ) ) && + ( + needs.classify-patchwork.result == 'success' || + needs.classify-patchwork.result == 'skipped' + ) && ( ( github.event_name == 'pull_request' && @@ -221,6 +290,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} + PATCHWORK_DETECTED: ${{ needs.classify-patchwork.outputs.patchwork == 'true' }} TRIM_CONC: >- ${{ github.event_name == 'pull_request' && @@ -263,14 +333,18 @@ jobs: CMD+=(--evals-only) fi - + PRIORITY_ARGS=() + if [ "$PATCHWORK_DETECTED" = "true" ]; then + PRIORITY_ARGS+=(--patchwork-detected) + fi CONFIG_JSON=$("${CMD[@]}") CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ --event-name "${{ github.event_name }}" \ --queue-namespace "${{ github.run_id }}" \ --labels-json "$PR_LABELS" \ - --pr-number "${{ github.event.pull_request.number || 0 }}") + --pr-number "${{ github.event.pull_request.number || 0 }}" \ + "${PRIORITY_ARGS[@]}") echo "search-space-config=$CONFIG_JSON" >> "$GITHUB_OUTPUT" python3 "${GITHUB_WORKSPACE}/utils/find_reusable_sweep_run.py" \ diff --git a/utils/ci_priority.py b/utils/ci_priority.py index c1ba379856..f19d49604f 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -23,6 +23,7 @@ class PriorityContext: labels: frozenset[str] = frozenset() queue_namespace: str = "local" pr_number: int | None = None + patchwork_detected: bool = False def _decimal(value: Any) -> Decimal: @@ -54,7 +55,10 @@ def calculate_priority( patchwork = policy["labels"]["patchwork"] patch_labels = set(patchwork["names"]) waiver_labels = set(patchwork.get("waived-by", [])) - if context.labels & patch_labels and not context.labels & waiver_labels: + if ( + (context.patchwork_detected or context.labels & patch_labels) + and not context.labels & waiver_labels + ): return _decimal(patchwork["score"]).quantize(SCORE_QUANTUM, ROUND_HALF_UP) adjustments = policy["adjustments"] @@ -145,6 +149,7 @@ def main() -> int: parser.add_argument("--labels-json", default="[]") parser.add_argument("--queue-namespace", default="local") parser.add_argument("--pr-number", type=int) + parser.add_argument("--patchwork-detected", action="store_true") parser.add_argument( "--input", type=Path, @@ -158,6 +163,7 @@ def main() -> int: labels=_labels_from_json(args.labels_json), queue_namespace=args.queue_namespace, pr_number=args.pr_number, + patchwork_detected=args.patchwork_detected, ) source = args.input.read_text() if args.input else sys.stdin.read() payload = json.loads(source) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 6db35d6b90..cf53cf6bb8 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -88,6 +88,19 @@ def test_patchwork_label_forces_bottom_priority_without_waiver(): POLICY, PriorityContext(labels=frozenset({"ci-patchwork", "ci-patchwork-waived"})), ) > Decimal("0.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext(patchwork_detected=True), + ) == Decimal("0.000") + assert calculate_priority( + entry, + POLICY, + PriorityContext( + labels=frozenset({"ci-patchwork-waived"}), + patchwork_detected=True, + ), + ) > Decimal("0.000") def test_priority_labels_do_not_override_automatic_score(): From e4d3f14df52ec3c797ed871fd92d62cfff57e443 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:33:50 -0500 Subject: [PATCH 10/21] Classify all priority criteria with Fable --- .github/workflows/run-sweep.yml | 53 ++++++++++++----------- utils/ci_priority.py | 76 ++++++++++++++++++++++++++++++--- utils/test_ci_priority.py | 42 +++++++++++++++++- 3 files changed, 139 insertions(+), 32 deletions(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index ee94c299e8..8f0635167d 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -154,7 +154,7 @@ jobs: --ref "${{ github.ref }}" \ --workflow-id "run-sweep.yml" - classify-patchwork: + classify-priority: needs: check-changelog runs-on: ubuntu-latest if: >- @@ -169,14 +169,14 @@ jobs: contents: read pull-requests: read outputs: - patchwork: ${{ steps.result.outputs.patchwork }} + criteria: ${{ steps.result.outputs.criteria }} steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 - - name: Classify patchwork with Fable + - name: Classify priority criteria with Fable id: fable continue-on-error: true uses: anthropics/claude-code-action@v1 @@ -189,18 +189,25 @@ jobs: --model 'claude-fable-5' --max-turns 8 --allowedTools "Read,Glob,Grep,Bash(git diff:*)" - --json-schema '{"type":"object","properties":{"patchwork":{"type":"boolean"},"reason":{"type":"string"}},"required":["patchwork","reason"]}' + --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik2.6","kimik2.7","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' prompt: | Inspect this PR's diff against its base branch. - Return patchwork=true only when the benchmark uses modified - upstream inference-engine or runtime source. This includes - patch files, git apply, source mutation, monkey-patching, - local engine builds, forks, or images containing unmerged - engine changes. + Return every matching priority criterion: - Configuration and recipe changes using released or upstream - software are not patchwork. + - multi-node: separate prefill and decode workers + - agentic: agentic-coding or AgentX workload + - eval-only: evaluation without throughput measurement + - fp4: FP4 precision + - mtp, eagle, eagle3: speculative decoding method + - sglang, vllm, dynamo-vllm: runtime framework + - model criterion: matching configured model family + - checklist-complete: PR checklist is satisfied + - patchwork: modified upstream engine or runtime source + + Patchwork includes patch files, git apply, source mutation, + monkey-patching, local engine builds, forks, or images + containing unmerged engine changes. Return a concise reason grounded in the diff. @@ -210,17 +217,17 @@ jobs: env: OUTPUT: ${{ steps.fable.outputs.structured_output || '{}' }} run: | - if jq -e 'type == "object" and (.patchwork | type == "boolean")' \ + if jq -e 'type == "object" and (.criteria | type == "array")' \ >/dev/null <<<"$OUTPUT"; then - patchwork=$(jq -r '.patchwork' <<<"$OUTPUT") + criteria=$(jq -c '.criteria' <<<"$OUTPUT") else - patchwork=true + criteria='["patchwork"]' fi - echo "patchwork=$patchwork" >> "$GITHUB_OUTPUT" - echo "Fable patchwork classification: $patchwork" + echo "criteria=$criteria" >> "$GITHUB_OUTPUT" + echo "Fable priority criteria: $criteria" setup: - needs: [check-changelog, reuse-sweep-gate, classify-patchwork] + needs: [check-changelog, reuse-sweep-gate, classify-priority] runs-on: ubuntu-latest permissions: actions: read @@ -241,8 +248,8 @@ jobs: ) ) && ( - needs.classify-patchwork.result == 'success' || - needs.classify-patchwork.result == 'skipped' + needs.classify-priority.result == 'success' || + needs.classify-priority.result == 'skipped' ) && ( ( @@ -290,7 +297,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} - PATCHWORK_DETECTED: ${{ needs.classify-patchwork.outputs.patchwork == 'true' }} + FABLE_CRITERIA: ${{ needs.classify-priority.outputs.criteria }} TRIM_CONC: >- ${{ github.event_name == 'pull_request' && @@ -333,10 +340,6 @@ jobs: CMD+=(--evals-only) fi - PRIORITY_ARGS=() - if [ "$PATCHWORK_DETECTED" = "true" ]; then - PRIORITY_ARGS+=(--patchwork-detected) - fi CONFIG_JSON=$("${CMD[@]}") CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ @@ -344,7 +347,7 @@ jobs: --queue-namespace "${{ github.run_id }}" \ --labels-json "$PR_LABELS" \ --pr-number "${{ github.event.pull_request.number || 0 }}" \ - "${PRIORITY_ARGS[@]}") + --criteria-json "$FABLE_CRITERIA") echo "search-space-config=$CONFIG_JSON" >> "$GITHUB_OUTPUT" python3 "${GITHUB_WORKSPACE}/utils/find_reusable_sweep_run.py" \ diff --git a/utils/ci_priority.py b/utils/ci_priority.py index f19d49604f..9561f9a1c0 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -23,7 +23,7 @@ class PriorityContext: labels: frozenset[str] = frozenset() queue_namespace: str = "local" pr_number: int | None = None - patchwork_detected: bool = False + criteria: frozenset[str] | None = None def _decimal(value: Any) -> Decimal: @@ -46,6 +46,52 @@ def _first_prefix_adjustment(value: str, adjustments: dict[str, Any]) -> Decimal +def _criterion_value(criteria: frozenset[str], choices: tuple[str, ...]) -> str: + matches = criteria.intersection(choices) + if len(matches) > 1: + raise ValueError(f"Conflicting CI priority criteria: {sorted(matches)}") + return next(iter(matches), "") + + +def _entry_from_criteria( + criteria: frozenset[str], + policy: dict[str, Any], +) -> dict[str, Any]: + fixed = { + "multi-node", + "agentic", + "eval-only", + "fp4", + "mtp", + "eagle", + "eagle3", + "sglang", + "vllm", + "dynamo-vllm", + "checklist-complete", + "patchwork", + } + allowed = fixed | set(policy["adjustments"].get("model-prefix", {})) + unknown = criteria - allowed + if unknown: + raise ValueError(f"Unknown CI priority criteria: {sorted(unknown)}") + return { + "prefill": {} if "multi-node" in criteria else None, + "scenario-type": "agentic-coding" if "agentic" in criteria else "", + "eval-only": "eval-only" in criteria, + "precision": "fp4" if "fp4" in criteria else "", + "spec-decoding": _criterion_value(criteria, ("mtp", "eagle", "eagle3")), + "framework": _criterion_value( + criteria, + ("sglang", "vllm", "dynamo-vllm"), + ), + "model-prefix": _criterion_value( + criteria, + tuple(policy["adjustments"].get("model-prefix", {})), + ), + } + + def calculate_priority( entry: dict[str, Any], policy: dict[str, Any], @@ -55,12 +101,18 @@ def calculate_priority( patchwork = policy["labels"]["patchwork"] patch_labels = set(patchwork["names"]) waiver_labels = set(patchwork.get("waived-by", [])) + criteria = context.criteria if ( - (context.patchwork_detected or context.labels & patch_labels) + ( + (criteria is not None and "patchwork" in criteria) + or context.labels & patch_labels + ) and not context.labels & waiver_labels ): return _decimal(patchwork["score"]).quantize(SCORE_QUANTUM, ROUND_HALF_UP) + if criteria is not None: + entry = _entry_from_criteria(criteria, policy) adjustments = policy["adjustments"] score = _decimal(policy["base-score"]) score += _decimal(adjustments.get("event", {}).get(context.event_name, 0)) @@ -84,7 +136,10 @@ def calculate_priority( ) checklist = policy["labels"].get("checklist-complete", {}) - if context.labels & set(checklist.get("names", [])): + if ( + (criteria is not None and "checklist-complete" in criteria) + or (criteria is None and context.labels & set(checklist.get("names", []))) + ): score += _decimal(checklist.get("adjustment", 0)) return score.quantize(SCORE_QUANTUM, ROUND_HALF_UP) @@ -142,6 +197,17 @@ def _labels_from_json(raw_labels: str) -> frozenset[str]: return frozenset(value) +def _criteria_from_json(raw_criteria: str) -> frozenset[str] | None: + if not raw_criteria: + return None + value = json.loads(raw_criteria) + if value is None: + return None + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError("--criteria-json must be a JSON array of strings") + return frozenset(value) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--policy", default="configs/ci-priority.yaml") @@ -149,7 +215,7 @@ def main() -> int: parser.add_argument("--labels-json", default="[]") parser.add_argument("--queue-namespace", default="local") parser.add_argument("--pr-number", type=int) - parser.add_argument("--patchwork-detected", action="store_true") + parser.add_argument("--criteria-json", default="") parser.add_argument( "--input", type=Path, @@ -163,7 +229,7 @@ def main() -> int: labels=_labels_from_json(args.labels_json), queue_namespace=args.queue_namespace, pr_number=args.pr_number, - patchwork_detected=args.patchwork_detected, + criteria=_criteria_from_json(args.criteria_json), ) source = args.input.read_text() if args.input else sys.stdin.read() payload = json.loads(source) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index cf53cf6bb8..2538d6bda4 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -2,6 +2,8 @@ from decimal import Decimal from pathlib import Path +import pytest + from ci_priority import PriorityContext, annotate_jobs, calculate_priority, load_policy @@ -91,18 +93,54 @@ def test_patchwork_label_forces_bottom_priority_without_waiver(): assert calculate_priority( entry, POLICY, - PriorityContext(patchwork_detected=True), + PriorityContext(criteria=frozenset({"patchwork"})), ) == Decimal("0.000") assert calculate_priority( entry, POLICY, PriorityContext( labels=frozenset({"ci-patchwork-waived"}), - patchwork_detected=True, + criteria=frozenset({"patchwork"}), ), ) > Decimal("0.000") +def test_fable_criteria_drive_all_configured_adjustments(): + entry = {"runner": "h100", "framework": "trt"} + criteria = frozenset({"multi-node", "agentic", "fp4", "mtp", "vllm", "dsr1"}) + equivalent_entry = { + "prefill": {}, + "scenario-type": "agentic-coding", + "precision": "fp4", + "spec-decoding": "mtp", + "framework": "vllm", + "model-prefix": "dsr1", + } + + assert calculate_priority( + entry, + POLICY, + PriorityContext(criteria=criteria), + ) == calculate_priority(equivalent_entry, POLICY) + + +def test_fable_criteria_reject_unknown_or_conflicting_values(): + entry = {"runner": "h100", "framework": "trt"} + + with pytest.raises(ValueError, match="Unknown CI priority criteria"): + calculate_priority( + entry, + POLICY, + PriorityContext(criteria=frozenset({"unknown"})), + ) + with pytest.raises(ValueError, match="Conflicting CI priority criteria"): + calculate_priority( + entry, + POLICY, + PriorityContext(criteria=frozenset({"vllm", "sglang"})), + ) + + def test_priority_labels_do_not_override_automatic_score(): entry = {"runner": "h100", "framework": "trt"} labels = frozenset( From aea0f1d3db698ab811bedd0d6819e601269f83d4 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:36:47 -0500 Subject: [PATCH 11/21] Apply Fable criteria per matrix job --- utils/ci_priority.py | 62 ++++++++++++++++++++++++++------------- utils/test_ci_priority.py | 23 +++++++++------ 2 files changed, 56 insertions(+), 29 deletions(-) diff --git a/utils/ci_priority.py b/utils/ci_priority.py index 9561f9a1c0..0a1b9daa53 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -45,17 +45,10 @@ def _first_prefix_adjustment(value: str, adjustments: dict[str, Any]) -> Decimal return Decimal(0) - -def _criterion_value(criteria: frozenset[str], choices: tuple[str, ...]) -> str: - matches = criteria.intersection(choices) - if len(matches) > 1: - raise ValueError(f"Conflicting CI priority criteria: {sorted(matches)}") - return next(iter(matches), "") - - def _entry_from_criteria( criteria: frozenset[str], policy: dict[str, Any], + entry: dict[str, Any], ) -> dict[str, Any]: fixed = { "multi-node", @@ -75,19 +68,48 @@ def _entry_from_criteria( unknown = criteria - allowed if unknown: raise ValueError(f"Unknown CI priority criteria: {sorted(unknown)}") + spec_decoding = str(entry.get("spec-decoding", "")) + framework = str(entry.get("framework", "")) + model_prefix = str(entry.get("model-prefix", "")) + framework_criteria = ("sglang", "vllm", "dynamo-vllm") + model_criteria = tuple(policy["adjustments"].get("model-prefix", {})) return { - "prefill": {} if "multi-node" in criteria else None, - "scenario-type": "agentic-coding" if "agentic" in criteria else "", - "eval-only": "eval-only" in criteria, - "precision": "fp4" if "fp4" in criteria else "", - "spec-decoding": _criterion_value(criteria, ("mtp", "eagle", "eagle3")), - "framework": _criterion_value( - criteria, - ("sglang", "vllm", "dynamo-vllm"), + "prefill": ( + {} if "multi-node" in criteria and entry.get("prefill") is not None else None + ), + "scenario-type": ( + "agentic-coding" + if "agentic" in criteria and entry.get("scenario-type") == "agentic-coding" + else "" + ), + "eval-only": "eval-only" in criteria and entry.get("eval-only") is True, + "precision": ( + "fp4" if "fp4" in criteria and entry.get("precision") == "fp4" else "" + ), + "spec-decoding": spec_decoding if spec_decoding in criteria else "", + "framework": ( + framework + if any( + criterion in criteria + and ( + framework == criterion + or framework.startswith(f"{criterion}-") + ) + for criterion in framework_criteria + ) + else "" ), - "model-prefix": _criterion_value( - criteria, - tuple(policy["adjustments"].get("model-prefix", {})), + "model-prefix": ( + model_prefix + if any( + criterion in criteria + and ( + model_prefix == criterion + or model_prefix.startswith(f"{criterion}-") + ) + for criterion in model_criteria + ) + else "" ), } @@ -112,7 +134,7 @@ def calculate_priority( return _decimal(patchwork["score"]).quantize(SCORE_QUANTUM, ROUND_HALF_UP) if criteria is not None: - entry = _entry_from_criteria(criteria, policy) + entry = _entry_from_criteria(criteria, policy, entry) adjustments = policy["adjustments"] score = _decimal(policy["base-score"]) score += _decimal(adjustments.get("event", {}).get(context.event_name, 0)) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 2538d6bda4..14afd7bd44 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -106,7 +106,6 @@ def test_patchwork_label_forces_bottom_priority_without_waiver(): def test_fable_criteria_drive_all_configured_adjustments(): - entry = {"runner": "h100", "framework": "trt"} criteria = frozenset({"multi-node", "agentic", "fp4", "mtp", "vllm", "dsr1"}) equivalent_entry = { "prefill": {}, @@ -116,16 +115,23 @@ def test_fable_criteria_drive_all_configured_adjustments(): "framework": "vllm", "model-prefix": "dsr1", } + entry = dict(equivalent_entry) assert calculate_priority( entry, POLICY, PriorityContext(criteria=criteria), ) == calculate_priority(equivalent_entry, POLICY) + unrelated_entry = {"runner": "h100", "framework": "trt"} + assert calculate_priority( + unrelated_entry, + POLICY, + PriorityContext(criteria=criteria), + ) == Decimal("1.000") -def test_fable_criteria_reject_unknown_or_conflicting_values(): - entry = {"runner": "h100", "framework": "trt"} +def test_fable_criteria_reject_unknown_values_and_allow_mixed_jobs(): + entry = {"runner": "h100", "framework": "vllm"} with pytest.raises(ValueError, match="Unknown CI priority criteria"): calculate_priority( @@ -133,12 +139,11 @@ def test_fable_criteria_reject_unknown_or_conflicting_values(): POLICY, PriorityContext(criteria=frozenset({"unknown"})), ) - with pytest.raises(ValueError, match="Conflicting CI priority criteria"): - calculate_priority( - entry, - POLICY, - PriorityContext(criteria=frozenset({"vllm", "sglang"})), - ) + assert calculate_priority( + entry, + POLICY, + PriorityContext(criteria=frozenset({"vllm", "sglang"})), + ) == Decimal("1.500") def test_priority_labels_do_not_override_automatic_score(): From 30f0ae8427651e6aa23fd3f5984d071b2cf4abc4 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:49:22 -0500 Subject: [PATCH 12/21] Model priority classification in sweep gating tests --- .../changelog_gate_tests/test_run_sweep_gating.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/utils/changelog_gate_tests/test_run_sweep_gating.py b/utils/changelog_gate_tests/test_run_sweep_gating.py index a36f1eb726..ecc3a5634d 100644 --- a/utils/changelog_gate_tests/test_run_sweep_gating.py +++ b/utils/changelog_gate_tests/test_run_sweep_gating.py @@ -2,10 +2,10 @@ The simulation jobs in `.github/workflows/test-changelog-gate.yml` hand-copy two of the gating `if` conditions and exercise two scenarios. This test parses -the real `check-changelog` -> `reuse-sweep-gate` -> `setup` conditions out of -`run-sweep.yml` and evaluates them with a minimal GitHub Actions expression -engine, so it cannot drift from production and it covers every distinct -skip/run decision. +the real `check-changelog` -> `reuse-sweep-gate` / `classify-priority` -> +`setup` conditions out of `run-sweep.yml` and evaluates them with a minimal +GitHub Actions expression engine, so it cannot drift from production and covers +every distinct skip/run decision. """ from __future__ import annotations @@ -25,6 +25,7 @@ ) CHECK_IF = _WF["jobs"]["check-changelog"]["if"] GATE_IF = _WF["jobs"]["reuse-sweep-gate"]["if"] +CLASSIFY_IF = _WF["jobs"]["classify-priority"]["if"] SETUP_IF = _WF["jobs"]["setup"]["if"] PR_TYPES = set(_WF["on"]["pull_request"]["types"]) @@ -187,7 +188,7 @@ def _eval(expr: str, ctx: dict) -> bool: # -------------------------------------------------------------------------- -# DAG evaluation: check-changelog -> reuse-sweep-gate -> setup +# DAG evaluation: check-changelog -> reuse-sweep-gate / classify-priority -> setup # -------------------------------------------------------------------------- def _ctx(sc: dict) -> dict: return { @@ -223,6 +224,9 @@ def run_dag(sc: dict) -> tuple[str, str, str]: ctx["needs.reuse-sweep-gate.result"] = gate_result ctx["needs.reuse-sweep-gate.outputs.skip-pr-sweep"] = skip + classify_result = "success" if _eval(CLASSIFY_IF, ctx) else "skipped" + ctx["needs.classify-priority.result"] = classify_result + setup = "RUN" if _eval(SETUP_IF, ctx) else "SKIP" return check_result, gate_result, setup From dea2f50181f89bebdde6586b2d67a8c49eaeab2c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 17:52:11 -0500 Subject: [PATCH 13/21] Update sweep dependency validation for priority classification --- utils/changelog_gate_tests/test_validate_perf_changelog.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/changelog_gate_tests/test_validate_perf_changelog.py b/utils/changelog_gate_tests/test_validate_perf_changelog.py index d6b6a3565f..5a4b67e348 100644 --- a/utils/changelog_gate_tests/test_validate_perf_changelog.py +++ b/utils/changelog_gate_tests/test_validate_perf_changelog.py @@ -320,9 +320,11 @@ def test_run_sweep_checks_changelog_before_reuse_and_setup() -> None: == "${{ steps.sweep_policy.outputs.skip-pr-sweep }}" ) assert jobs["reuse-sweep-gate"]["needs"] == "check-changelog" + assert jobs["classify-priority"]["needs"] == "check-changelog" assert jobs["setup"]["needs"] == [ "check-changelog", "reuse-sweep-gate", + "classify-priority", ] assert ( "needs.check-changelog.result == 'success'" From 6bde37f2ece54c3de66e6c84f50c8248ecc9477b Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:58:38 -0500 Subject: [PATCH 14/21] Combine CI priority and queue labels --- .../workflows/benchmark-multinode-tmpl.yml | 4 +- .github/workflows/benchmark-tmpl.yml | 4 +- .github/workflows/collectivex-sweep.yml | 12 +++- .github/workflows/profile.yml | 2 +- .github/workflows/speedbench-al.yml | 9 ++- configs/ci-priority.yaml | 2 +- utils/ci_priority.py | 2 +- utils/ci_priority_controller.py | 64 ++++++++----------- utils/test_ci_priority.py | 2 +- utils/test_ci_priority_controller.py | 40 +++++++----- 10 files changed, 71 insertions(+), 70 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index ee9241dc3e..275dccaaa0 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -250,14 +250,14 @@ jobs: ( inputs.skip-queue-pr != '' && format( - '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}","ci-skip-queue-pr-{3}"]', + '["self-hosted","{0}","ci-job-{1}-{2}","ci-skip-queue-pr-{3}"]', inputs.runner, inputs.priority, inputs.queue-token, inputs.skip-queue-pr ) || format( - '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}"]', + '["self-hosted","{0}","ci-job-{1}-{2}"]', inputs.runner, inputs.priority, inputs.queue-token diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 5451c8cc46..e9e31f9d96 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -180,14 +180,14 @@ jobs: ( inputs.skip-queue-pr != '' && format( - '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}","ci-skip-queue-pr-{3}"]', + '["self-hosted","{0}","ci-job-{1}-{2}","ci-skip-queue-pr-{3}"]', inputs.runner, inputs.priority, inputs.queue-token, inputs.skip-queue-pr ) || format( - '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}"]', + '["self-hosted","{0}","ci-job-{1}-{2}"]', inputs.runner, inputs.priority, inputs.queue-token diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml index e7994b7569..b326e9cdc1 100644 --- a/.github/workflows/collectivex-sweep.yml +++ b/.github/workflows/collectivex-sweep.yml @@ -50,6 +50,7 @@ jobs: INPUT_ONLY_SKU: ${{ inputs.only_sku }} INPUT_EXCLUDE_SKUS: ${{ inputs.exclude_skus }} INPUT_EP_SIZES: ${{ inputs.ep_sizes }} + RUN_ID: ${{ github.run_id }} run: | set -euo pipefail args=(--backend "$INPUT_BACKEND") @@ -58,12 +59,18 @@ jobs: [ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES") python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null python3 - "$GITHUB_OUTPUT" <<'PY' + import hashlib import json + import os import pathlib import sys matrix = json.loads(pathlib.Path("matrix_full.json").read_text()) cells = matrix["include"] + for index, cell in enumerate(cells): + canonical = json.dumps(cell, sort_keys=True, separators=(",", ":")) + material = f"{os.environ['RUN_ID']}:{index}:{canonical}".encode() + cell["queue-token"] = hashlib.sha256(material).hexdigest()[:32] slim = {"include": cells} with open(sys.argv[1], "a", encoding="utf-8") as output: output.write(f"matrix={json.dumps(slim, separators=(',', ':'))}\n") @@ -93,11 +100,10 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( - '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}-{3}"]', + '["self-hosted","{0}","ci-job-{1}-{2}"]', matrix.sku, needs.setup.outputs.priority, - github.run_id, - matrix.id + matrix.queue-token ) || format('["{0}"]', matrix.sku) ) }} diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index a505b196bb..b326af5e7e 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -110,7 +110,7 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( - '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}"]', + '["self-hosted","{0}","ci-job-{1}-{2}"]', matrix.config.runner, matrix.config.priority, matrix.config['queue-token'] diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index e5b5fa1f72..bc9f01fab5 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -107,6 +107,7 @@ jobs: runs-on: ubuntu-latest outputs: priority: ${{ steps.score.outputs.priority }} + queue-token: ${{ steps.score.outputs.queue-token }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: { clean: true, persist-credentials: false } @@ -125,8 +126,10 @@ jobs: precision: "fp4", "spec-decoding": "mtp" }]') - scored=$(printf '%s' "$entry" | python3 utils/ci_priority.py) + scored=$(printf '%s' "$entry" | + python3 utils/ci_priority.py --queue-namespace "${{ github.run_id }}") echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" + echo "queue-token=$(jq -r '.[0][\"queue-token\"]' <<<"$scored")" >> "$GITHUB_OUTPUT" collect-al: needs: setup @@ -134,10 +137,10 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( - '["self-hosted","{0}","ci-priority-{1}","ci-queue-{2}"]', + '["self-hosted","{0}","ci-job-{1}-{2}"]', inputs.runner, needs.setup.outputs.priority, - github.run_id + needs.setup.outputs.queue-token ) || format('["{0}"]', inputs.runner) ) }} diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index 10d3c1b290..a65daf842b 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -1,7 +1,7 @@ version: 1 # Higher scores run first. Scores are uncapped and remain human-readable in -# workflow job labels (for example, ci-priority-2.750). +# workflow job labels (for example, ci-job-2.750-). base-score: 1.0 adjustments: diff --git a/utils/ci_priority.py b/utils/ci_priority.py index 0a1b9daa53..39501bc885 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -174,7 +174,7 @@ def format_priority(score: Decimal) -> str: def queue_token(value: dict[str, Any], namespace: str, path: tuple[str, ...]) -> str: canonical = json.dumps(value, sort_keys=True, separators=(",", ":")) material = f"{namespace}:{'/'.join(path)}:{canonical}".encode() - return hashlib.sha256(material).hexdigest()[:20] + return hashlib.sha256(material).hexdigest()[:32] def annotate_jobs( diff --git a/utils/ci_priority_controller.py b/utils/ci_priority_controller.py index 65c8c7a497..2a20970d9d 100755 --- a/utils/ci_priority_controller.py +++ b/utils/ci_priority_controller.py @@ -22,13 +22,11 @@ from authorize_skip_queue import is_authorized -PRIORITY_LABEL_PREFIX = "ci-priority-" -QUEUE_LABEL_PREFIX = "ci-queue-" +JOB_LABEL_PREFIX = "ci-job-" SKIP_QUEUE_LABEL_PREFIX = "ci-skip-queue-pr-" -PRIORITY_LABEL_RE = re.compile( - r"^ci-priority-(?P[0-9]+(?:\.[0-9]+)?)$" +JOB_LABEL_RE = re.compile( + r"^ci-job-(?P[0-9]+(?:\.[0-9]+)?)-(?P[0-9a-f]{32})$" ) -QUEUE_LABEL_RE = re.compile(r"^ci-queue-(?P[a-zA-Z0-9._-]+)$") SKIP_QUEUE_LABEL_RE = re.compile(r"^ci-skip-queue-pr-(?P[1-9][0-9]*)$") @@ -39,28 +37,20 @@ def parse_timestamp(value: str | None) -> datetime: return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) -def priority_from_labels(labels: Iterable[str]) -> tuple[str, Decimal] | None: - candidates = [label for label in labels if label.startswith(PRIORITY_LABEL_PREFIX)] - invalid = [label for label in candidates if not PRIORITY_LABEL_RE.fullmatch(label)] +def job_label_from_labels( + labels: Iterable[str], +) -> tuple[str, Decimal, str] | None: + candidates = [label for label in labels if label.startswith(JOB_LABEL_PREFIX)] + invalid = [label for label in candidates if not JOB_LABEL_RE.fullmatch(label)] if invalid: - raise ValueError(f"Job has invalid CI priority labels: {invalid}") + raise ValueError(f"Job has invalid CI job labels: {invalid}") if len(candidates) > 1: - raise ValueError(f"Job has multiple CI priority labels: {candidates}") + raise ValueError(f"Job has multiple CI job labels: {candidates}") if not candidates: return None label = candidates[0] - match = PRIORITY_LABEL_RE.fullmatch(label) - return label, Decimal(match.group("score")) - - -def queue_label_from_labels(labels: Iterable[str]) -> str | None: - candidates = [label for label in labels if label.startswith(QUEUE_LABEL_PREFIX)] - invalid = [label for label in candidates if not QUEUE_LABEL_RE.fullmatch(label)] - if invalid: - raise ValueError(f"Job has invalid CI queue labels: {invalid}") - if len(candidates) > 1: - raise ValueError(f"Job has multiple CI queue labels: {candidates}") - return candidates[0] if candidates else None + match = JOB_LABEL_RE.fullmatch(label) + return label, Decimal(match.group("score")), match.group("token") def skip_queue_from_labels(labels: Iterable[str]) -> tuple[str, int] | None: candidates = [label for label in labels if label.startswith(SKIP_QUEUE_LABEL_PREFIX)] @@ -80,9 +70,7 @@ def without_scheduling_labels(labels: Iterable[str]) -> frozenset[str]: return frozenset( label for label in labels - if not label.startswith( - (PRIORITY_LABEL_PREFIX, QUEUE_LABEL_PREFIX, SKIP_QUEUE_LABEL_PREFIX) - ) + if not label.startswith((JOB_LABEL_PREFIX, SKIP_QUEUE_LABEL_PREFIX)) ) @@ -149,13 +137,13 @@ def effective_priority( aging_per_hour: Decimal, authorized_skip_prs: frozenset[int] = frozenset(), ) -> Decimal: - priority = priority_from_labels(job.labels) - if priority is None: - raise ValueError(f"Job {job.id} has no CI priority label") + job_label = job_label_from_labels(job.labels) + if job_label is None: + raise ValueError(f"Job {job.id} has no CI job label") skip_request = skip_queue_from_labels(job.labels) if skip_request is not None and skip_request[1] in authorized_skip_prs: return Decimal("Infinity") - _, score = priority + _, score, _ = job_label waited_hours = Decimal(str(max(0.0, (now - job.queued_at).total_seconds()) / 3600)) return score + waited_hours * aging_per_hour @@ -176,17 +164,16 @@ def plan_label_updates( ) -> list[LabelUpdate]: """Assign each idle runner to the best compatible queued job. - Busy and offline runners are never relabeled. Priority and one-shot queue - labels are removed from unassigned idle runners, so a runner cannot take a + Busy and offline runners are never relabeled. One-shot job labels are + removed from unassigned idle runners, so a runner cannot take a second job before the controller has considered newly queued higher priorities. """ eligible_jobs = [] for job in jobs: - priority = priority_from_labels(job.labels) - queue_label = queue_label_from_labels(job.labels) + job_label = job_label_from_labels(job.labels) skip_queue_from_labels(job.labels) - if priority is not None and queue_label is not None: + if job_label is not None: eligible_jobs.append(job) eligible_jobs.sort( key=lambda job: ( @@ -209,12 +196,11 @@ def plan_label_updates( } for index, job in enumerate(eligible_jobs): - priority = priority_from_labels(job.labels) - queue_label = queue_label_from_labels(job.labels) + job_label = job_label_from_labels(job.labels) skip_request = skip_queue_from_labels(job.labels) - if priority is None or queue_label is None: + if job_label is None: continue - priority_label, _ = priority + scheduling_label, _, _ = job_label candidates = [ runner for runner in remaining.values() if is_compatible(job, runner) ] @@ -231,7 +217,7 @@ def plan_label_updates( ) desired[runner.id] = ( without_scheduling_labels(runner.labels) - | {priority_label, queue_label} + | {scheduling_label} | ({skip_request[0]} if skip_request is not None else set()), job.id, ) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 14afd7bd44..4556efc96e 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -178,7 +178,7 @@ def test_annotation_only_touches_runnable_matrix_entries(): annotated = annotate_jobs(payload, POLICY) assert annotated["single_node"]["1k1k"][0]["priority"] == "3.750" - assert len(annotated["single_node"]["1k1k"][0]["queue-token"]) == 20 + assert len(annotated["single_node"]["1k1k"][0]["queue-token"]) == 32 assert "priority" not in annotated["changelog_metadata"] assert "priority" not in payload["single_node"]["1k1k"][0] assert "queue-token" not in payload["single_node"]["1k1k"][0] diff --git a/utils/test_ci_priority_controller.py b/utils/test_ci_priority_controller.py index 8f44c3ec0a..91c6a36dd2 100644 --- a/utils/test_ci_priority_controller.py +++ b/utils/test_ci_priority_controller.py @@ -17,6 +17,10 @@ NOW = datetime(2026, 7, 14, 18, 0, tzinfo=timezone.utc) +def job_label(job_id, priority): + return f"ci-job-{priority}-{job_id:032x}" + + def job(job_id, priority, hardware, queued_minutes=0): return QueuedJob( id=job_id, @@ -24,8 +28,7 @@ def job(job_id, priority, hardware, queued_minutes=0): labels=frozenset({ "self-hosted", hardware, - f"ci-priority-{priority}", - f"ci-queue-job-{job_id}", + job_label(job_id, priority), }), queued_at=NOW - timedelta(minutes=queued_minutes), name=f"job-{job_id}", @@ -72,10 +75,8 @@ def test_assigns_best_compatible_job_to_each_idle_runner(): ("b200_00", 2), ("h100_00", 1), ] - assert "ci-priority-0.700" in updates[0].labels - assert "ci-priority-5.000" in updates[1].labels - assert "ci-queue-job-2" in updates[0].labels - assert "ci-queue-job-1" in updates[1].labels + assert job_label(2, "0.700") in updates[0].labels + assert job_label(1, "5.000") in updates[1].labels def test_aging_breaks_starvation_between_nearby_priorities(): @@ -113,7 +114,7 @@ def test_authorized_skip_queue_outranks_numeric_priority(): ) assert updates[0].assigned_job_id == 2 - assert "ci-priority-1.000" in updates[0].labels + assert job_label(2, "1.000") in updates[0].labels assert "ci-skip-queue-pr-2124" in updates[0].labels @@ -181,8 +182,7 @@ def test_unused_idle_runner_loses_stale_priority_label(): 10, "b200_00", "b200", - "ci-priority-5.000", - "ci-queue-old-job", + job_label(99, "5.000"), ) ] @@ -190,14 +190,13 @@ def test_unused_idle_runner_loses_stale_priority_label(): assert len(updates) == 1 assert updates[0].assigned_job_id is None - assert all(not label.startswith("ci-priority-") for label in updates[0].labels) - assert all(not label.startswith("ci-queue-") for label in updates[0].labels) + assert all(not label.startswith("ci-job-") for label in updates[0].labels) def test_busy_and_offline_runners_are_never_relabelled(): runners = [ - runner(10, "b200_00", "b200", "ci-priority-5.000", busy=True), - runner(11, "b200_01", "b200", "ci-priority-5.000", status="offline"), + runner(10, "b200_00", "b200", job_label(98, "5.000"), busy=True), + runner(11, "b200_01", "b200", job_label(99, "5.000"), status="offline"), ] assert plan_label_updates([job(1, "0.700", "b200")], runners, now=NOW) == [] @@ -238,8 +237,16 @@ def test_preserves_scarce_runner_for_exact_job(): @pytest.mark.parametrize( ("valid", "invalid", "message"), [ - ("ci-priority-1.000", "ci-priority--1", "invalid CI priority"), - ("ci-queue-job-1", "ci-queue-", "invalid CI queue"), + ( + job_label(1, "1.000"), + "ci-job--1-00000000000000000000000000000001", + "invalid CI job", + ), + ( + job_label(1, "1.000"), + "ci-job-1.000-not-a-hash", + "invalid CI job", + ), ("ci-skip-queue-pr-2124", "ci-skip-queue-pr-zero", "invalid skip_queue"), ], ) @@ -280,8 +287,7 @@ def paged(self, path, key): "labels": [ "self-hosted", "b200", - "ci-priority-1.000", - "ci-queue-job-22", + job_label(22, "1.000"), ], }] raise AssertionError(path) From 0a1a08e4262b38348cf56224c7038747ab4de9cc Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 14 Jul 2026 21:13:49 -0500 Subject: [PATCH 15/21] Move priority controller to CI tracker --- configs/ci-priority.yaml | 9 +- utils/authorize_skip_queue.py | 143 --------- utils/ci_priority_controller.py | 458 --------------------------- utils/test_authorize_skip_queue.py | 92 ------ utils/test_ci_priority_controller.py | 301 ------------------ 5 files changed, 1 insertion(+), 1002 deletions(-) delete mode 100755 utils/authorize_skip_queue.py delete mode 100755 utils/ci_priority_controller.py delete mode 100644 utils/test_authorize_skip_queue.py delete mode 100644 utils/test_ci_priority_controller.py diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index a65daf842b..f72073f0c9 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -31,11 +31,9 @@ adjustments: dsr1: 0.75 labels: - # Controller verifies skip_queue labeling actor membership. + # inferencex-ci-tracker verifies the active actor with INFX_CI_SCHEDULER_PAT. skip-queue: name: skip_queue - organization: SemiAnalysisAI - team-slug: core checklist-complete: names: [ci-checklist-complete] adjustment: 0.25 @@ -44,8 +42,3 @@ labels: score: 0.0 waived-by: [ci-patchwork-waived] -scheduler: - # Waiting jobs improve by this many score points per hour. This prevents - # ordinary work from starving without allowing patchwork jobs to jump the - # queue in a realistic time window. - aging-per-hour: 0.25 diff --git a/utils/authorize_skip_queue.py b/utils/authorize_skip_queue.py deleted file mode 100755 index 415ed16a99..0000000000 --- a/utils/authorize_skip_queue.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -"""Verify that the active skip_queue label was applied by a trusted team member.""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import urllib.error -import urllib.request -from pathlib import Path -from typing import Any - -import yaml - - -class GitHubApi: - def __init__(self, token: str, api_url: str = "https://api.github.com"): - self.token = token - self.api_url = api_url.rstrip("/") - - def request(self, path: str) -> Any: - request = urllib.request.Request( - f"{self.api_url}{path}", - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {self.token}", - "X-GitHub-Api-Version": "2022-11-28", - }, - ) - try: - with urllib.request.urlopen(request, timeout=30) as response: - return json.loads(response.read()) - except urllib.error.HTTPError as error: - detail = error.read().decode(errors="replace") - raise RuntimeError(f"GitHub API GET {path} failed: {error.code} {detail}") from error - - def paged(self, path: str) -> list[dict[str, Any]]: - separator = "&" if "?" in path else "?" - values = [] - page = 1 - while True: - batch = self.request(f"{path}{separator}per_page=100&page={page}") - values.extend(batch) - if len(batch) < 100: - return values - page += 1 - - -def active_label_actor(events: list[dict[str, Any]], label_name: str) -> str | None: - """Return who applied the currently active instance of a label.""" - actor = None - for event in events: - if event.get("label", {}).get("name") != label_name: - continue - if event.get("event") == "labeled": - actor = event.get("actor", {}).get("login") - elif event.get("event") == "unlabeled": - actor = None - return actor - - -def is_authorized( - api: GitHubApi, - *, - repository: str, - pr_number: int, - organization: str, - team_slug: str, - label_name: str, -) -> tuple[bool, str | None]: - events = api.paged(f"/repos/{repository}/issues/{pr_number}/timeline") - actor = active_label_actor(events, label_name) - if not actor: - return False, None - - membership = api.request( - f"/orgs/{organization}/teams/{team_slug}/memberships/{actor}" - ) - return membership.get("state") == "active", actor - - -def load_skip_policy(path: str | Path) -> dict[str, Any]: - with Path(path).open() as policy_file: - policy = yaml.safe_load(policy_file) - return policy["labels"]["skip-queue"] - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repository", required=True) - parser.add_argument("--pr-number", required=True, type=int) - parser.add_argument( - "--policy", - type=Path, - default=Path(__file__).parents[1] / "configs" / "ci-priority.yaml", - ) - parser.add_argument("--organization") - parser.add_argument("--team-slug") - parser.add_argument("--label") - parser.add_argument("--token-env", default="REPO_PAT") - parser.add_argument("--api-url", default="https://api.github.com") - args = parser.parse_args() - skip_policy = load_skip_policy(args.policy) - organization = args.organization or skip_policy["organization"] - team_slug = args.team_slug or skip_policy["team-slug"] - label_name = args.label or skip_policy["name"] - - token = os.environ.get(args.token_env) - if not token: - print(f"::warning::{args.token_env} is unavailable; refusing skip_queue authorization", file=sys.stderr) - print("false") - return 0 - - try: - authorized, actor = is_authorized( - GitHubApi(token, args.api_url), - repository=args.repository, - pr_number=args.pr_number, - organization=organization, - team_slug=team_slug, - label_name=label_name, - ) - except RuntimeError as error: - print(f"::warning::{error}; refusing skip_queue authorization", file=sys.stderr) - print("false") - return 0 - - if authorized: - print(f"::notice::skip_queue authorized by {actor}", file=sys.stderr) - elif actor: - print( - f"::warning::skip_queue was applied by {actor}, who is not an active " - f"{organization}/{team_slug} member", - file=sys.stderr, - ) - print("true" if authorized else "false") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/utils/ci_priority_controller.py b/utils/ci_priority_controller.py deleted file mode 100755 index 2a20970d9d..0000000000 --- a/utils/ci_priority_controller.py +++ /dev/null @@ -1,458 +0,0 @@ -#!/usr/bin/env python3 -"""Plan or apply priority labels for idle self-hosted GitHub Actions runners.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import sys -import time -import urllib.error -import urllib.parse -import urllib.request -from dataclasses import dataclass -from datetime import datetime, timezone -from decimal import Decimal -from pathlib import Path -from typing import Any, Iterable - -import yaml - -from authorize_skip_queue import is_authorized - -JOB_LABEL_PREFIX = "ci-job-" -SKIP_QUEUE_LABEL_PREFIX = "ci-skip-queue-pr-" -JOB_LABEL_RE = re.compile( - r"^ci-job-(?P[0-9]+(?:\.[0-9]+)?)-(?P[0-9a-f]{32})$" -) -SKIP_QUEUE_LABEL_RE = re.compile(r"^ci-skip-queue-pr-(?P[1-9][0-9]*)$") - - -def parse_timestamp(value: str | None) -> datetime: - if not value: - return datetime.now(timezone.utc) - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) - - -def job_label_from_labels( - labels: Iterable[str], -) -> tuple[str, Decimal, str] | None: - candidates = [label for label in labels if label.startswith(JOB_LABEL_PREFIX)] - invalid = [label for label in candidates if not JOB_LABEL_RE.fullmatch(label)] - if invalid: - raise ValueError(f"Job has invalid CI job labels: {invalid}") - if len(candidates) > 1: - raise ValueError(f"Job has multiple CI job labels: {candidates}") - if not candidates: - return None - label = candidates[0] - match = JOB_LABEL_RE.fullmatch(label) - return label, Decimal(match.group("score")), match.group("token") - -def skip_queue_from_labels(labels: Iterable[str]) -> tuple[str, int] | None: - candidates = [label for label in labels if label.startswith(SKIP_QUEUE_LABEL_PREFIX)] - invalid = [label for label in candidates if not SKIP_QUEUE_LABEL_RE.fullmatch(label)] - if invalid: - raise ValueError(f"Job has invalid skip_queue labels: {invalid}") - if len(candidates) > 1: - raise ValueError(f"Job has multiple skip_queue labels: {candidates}") - if not candidates: - return None - label = candidates[0] - match = SKIP_QUEUE_LABEL_RE.fullmatch(label) - return label, int(match.group("number")) - - -def without_scheduling_labels(labels: Iterable[str]) -> frozenset[str]: - return frozenset( - label - for label in labels - if not label.startswith((JOB_LABEL_PREFIX, SKIP_QUEUE_LABEL_PREFIX)) - ) - - -@dataclass(frozen=True) -class QueuedJob: - id: int - run_id: int - labels: frozenset[str] - queued_at: datetime - name: str = "" - - @classmethod - def from_payload(cls, payload: dict[str, Any], run_id: int | None = None) -> "QueuedJob": - return cls( - id=int(payload["id"]), - run_id=int(run_id if run_id is not None else payload.get("run_id", 0)), - labels=frozenset(payload.get("labels", [])), - queued_at=parse_timestamp(payload.get("created_at") or payload.get("queued_at")), - name=str(payload.get("name", "")), - ) - - -@dataclass(frozen=True) -class Runner: - id: int - name: str - labels: frozenset[str] - status: str - busy: bool - - @classmethod - def from_payload(cls, payload: dict[str, Any]) -> "Runner": - raw_labels = payload.get("labels", []) - labels = [item["name"] if isinstance(item, dict) else item for item in raw_labels] - return cls( - id=int(payload["id"]), - name=str(payload["name"]), - labels=frozenset(labels), - status=str(payload.get("status", "offline")), - busy=bool(payload.get("busy", False)), - ) - - -@dataclass(frozen=True) -class LabelUpdate: - runner_id: int - runner_name: str - labels: tuple[str, ...] - assigned_job_id: int | None - - def as_dict(self) -> dict[str, Any]: - return { - "runner_id": self.runner_id, - "runner_name": self.runner_name, - "labels": list(self.labels), - "assigned_job_id": self.assigned_job_id, - } - - -def effective_priority( - job: QueuedJob, - *, - now: datetime, - aging_per_hour: Decimal, - authorized_skip_prs: frozenset[int] = frozenset(), -) -> Decimal: - job_label = job_label_from_labels(job.labels) - if job_label is None: - raise ValueError(f"Job {job.id} has no CI job label") - skip_request = skip_queue_from_labels(job.labels) - if skip_request is not None and skip_request[1] in authorized_skip_prs: - return Decimal("Infinity") - _, score, _ = job_label - waited_hours = Decimal(str(max(0.0, (now - job.queued_at).total_seconds()) / 3600)) - return score + waited_hours * aging_per_hour - - -def is_compatible(job: QueuedJob, runner: Runner) -> bool: - required = without_scheduling_labels(job.labels) - available = without_scheduling_labels(runner.labels) - return required.issubset(available) - - -def plan_label_updates( - jobs: Iterable[QueuedJob], - runners: Iterable[Runner], - *, - aging_per_hour: Decimal = Decimal("0.25"), - authorized_skip_prs: frozenset[int] = frozenset(), - now: datetime | None = None, -) -> list[LabelUpdate]: - """Assign each idle runner to the best compatible queued job. - - Busy and offline runners are never relabeled. One-shot job labels are - removed from unassigned idle runners, so a runner cannot take a - second job before the controller has considered newly queued higher - priorities. - """ - eligible_jobs = [] - for job in jobs: - job_label = job_label_from_labels(job.labels) - skip_queue_from_labels(job.labels) - if job_label is not None: - eligible_jobs.append(job) - eligible_jobs.sort( - key=lambda job: ( - -effective_priority( - job, - now=now, - aging_per_hour=aging_per_hour, - authorized_skip_prs=authorized_skip_prs, - ), - job.queued_at, - job.id, - ) - ) - - idle_runners = [runner for runner in runners if runner.status == "online" and not runner.busy] - remaining = {runner.id: runner for runner in idle_runners} - desired: dict[int, tuple[frozenset[str], int | None]] = { - runner.id: (without_scheduling_labels(runner.labels), None) - for runner in idle_runners - } - - for index, job in enumerate(eligible_jobs): - job_label = job_label_from_labels(job.labels) - skip_request = skip_queue_from_labels(job.labels) - if job_label is None: - continue - scheduling_label, _, _ = job_label - candidates = [ - runner for runner in remaining.values() if is_compatible(job, runner) - ] - if not candidates: - continue - later_jobs = eligible_jobs[index + 1:] - runner = min( - candidates, - key=lambda candidate: ( - sum(is_compatible(later, candidate) for later in later_jobs), - candidate.name, - candidate.id, - ), - ) - desired[runner.id] = ( - without_scheduling_labels(runner.labels) - | {scheduling_label} - | ({skip_request[0]} if skip_request is not None else set()), - job.id, - ) - del remaining[runner.id] - - updates = [] - for runner in sorted(idle_runners, key=lambda item: (item.name, item.id)): - labels, job_id = desired[runner.id] - if labels != runner.labels: - updates.append( - LabelUpdate( - runner_id=runner.id, - runner_name=runner.name, - labels=tuple(sorted(labels)), - assigned_job_id=job_id, - ) - ) - return updates - - -class GitHubClient: - def __init__(self, repository: str, token: str, api_url: str = "https://api.github.com"): - if repository.count("/") != 1: - raise ValueError("Repository must be OWNER/REPO") - self.repository = repository - self.token = token - self.api_url = api_url.rstrip("/") - - def request(self, method: str, path: str, body: Any = None) -> Any: - data = None if body is None else json.dumps(body).encode() - request = urllib.request.Request( - f"{self.api_url}{path}", - data=data, - method=method, - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {self.token}", - "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28", - }, - ) - try: - with urllib.request.urlopen(request, timeout=30) as response: - raw = response.read() - except urllib.error.HTTPError as error: - detail = error.read().decode(errors="replace") - raise RuntimeError(f"GitHub API {method} {path} failed: {error.code} {detail}") from error - except urllib.error.URLError as error: - raise RuntimeError(f"GitHub API {method} {path} failed: {error}") from error - return json.loads(raw) if raw else None - - def paged( - self, - path: str, - key: str | None, - ) -> list[dict[str, Any]]: - separator = "&" if "?" in path else "?" - values = [] - page = 1 - while True: - payload = self.request("GET", f"{path}{separator}per_page=100&page={page}") - batch = payload if key is None else payload[key] - values.extend(batch) - if len(batch) < 100: - return values - page += 1 - - def queued_jobs(self) -> list[QueuedJob]: - base = f"/repos/{self.repository}" - runs_by_id = {} - for status in ("queued", "in_progress"): - runs = self.paged( - f"{base}/actions/runs?status={status}", - "workflow_runs", - ) - runs_by_id.update({int(run["id"]): run for run in runs}) - - jobs = [] - for run_id in sorted(runs_by_id): - run_jobs = self.paged( - f"{base}/actions/runs/{run_id}/jobs?filter=latest", - "jobs", - ) - jobs.extend( - QueuedJob.from_payload(job, run_id) - for job in run_jobs - if job.get("status") == "queued" - ) - return jobs - - def runners(self) -> list[Runner]: - payloads = self.paged(f"/repos/{self.repository}/actions/runners", "runners") - return [Runner.from_payload(payload) for payload in payloads] - - def replace_runner_labels(self, runner_id: int, labels: Iterable[str]) -> None: - self.request( - "PUT", - f"/repos/{self.repository}/actions/runners/{runner_id}/labels", - {"labels": sorted(labels)}, - ) - - -class AuthorizationApi: - def __init__(self, client: GitHubClient): - self.client = client - - def paged(self, path: str) -> list[dict[str, Any]]: - return self.client.paged(path, None) - - def request(self, path: str) -> Any: - return self.client.request("GET", path) - - -def load_policy(policy_path: str | Path) -> dict[str, Any]: - with Path(policy_path).open() as policy_file: - return yaml.safe_load(policy_file) - - -def authorize_skip_requests( - client: GitHubClient, - jobs: Iterable[QueuedJob], - policy: dict[str, Any], -) -> frozenset[int]: - requests = { - request[1] - for job in jobs - if (request := skip_queue_from_labels(job.labels)) is not None - } - skip_policy = policy["labels"]["skip-queue"] - api = AuthorizationApi(client) - authorized = set() - for pr_number in sorted(requests): - try: - allowed, actor = is_authorized( - api, - repository=client.repository, - pr_number=pr_number, - organization=skip_policy["organization"], - team_slug=skip_policy["team-slug"], - label_name=skip_policy["name"], - ) - except RuntimeError as error: - print(f"::warning::{error}; refusing skip_queue authorization", file=sys.stderr) - continue - if allowed: - print(f"::notice::skip_queue authorized by {actor}", file=sys.stderr) - authorized.add(pr_number) - return frozenset(authorized) - - -def load_aging_rate(policy: dict[str, Any]) -> Decimal: - return Decimal(str(policy["scheduler"]["aging-per-hour"])) - - -def reconcile( - client: GitHubClient, - policy: dict[str, Any], - apply: bool, -) -> list[LabelUpdate]: - jobs = client.queued_jobs() - updates = plan_label_updates( - jobs, - client.runners(), - aging_per_hour=load_aging_rate(policy), - authorized_skip_prs=authorize_skip_requests(client, jobs, policy), - ) - if apply: - for update in updates: - client.replace_runner_labels(update.runner_id, update.labels) - return updates - - -def _load_json(path: Path) -> Any: - with path.open() as source: - return json.load(source) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--policy", default="configs/ci-priority.yaml") - subparsers = parser.add_subparsers(dest="command", required=True) - - plan_parser = subparsers.add_parser("plan", help="Plan from local API response fixtures") - plan_parser.add_argument("--jobs", type=Path, required=True) - plan_parser.add_argument("--runners", type=Path, required=True) - plan_parser.add_argument("--now", help="ISO-8601 clock for deterministic simulations") - plan_parser.add_argument("--authorized-skip-pr", action="append", type=int, default=[]) - - reconcile_parser = subparsers.add_parser("reconcile", help="Poll GitHub and reconcile runner labels") - reconcile_parser.add_argument("--repository", required=True) - reconcile_parser.add_argument("--api-url", default="https://api.github.com") - # REPO_PAT needs Actions and Issues read. - # REPO_PAT needs Administration write and Members read. - reconcile_parser.add_argument("--token-env", default="REPO_PAT") - reconcile_parser.add_argument("--apply", action="store_true") - reconcile_parser.add_argument("--watch", type=float, default=0, metavar="SECONDS") - - args = parser.parse_args() - policy = load_policy(args.policy) - - if args.command == "plan": - raw_jobs = _load_json(args.jobs) - raw_runners = _load_json(args.runners) - jobs = [QueuedJob.from_payload(job) for job in raw_jobs] - runners = [Runner.from_payload(runner) for runner in raw_runners] - now = parse_timestamp(args.now) if args.now else None - updates = plan_label_updates( - jobs, - runners, - aging_per_hour=load_aging_rate(policy), - authorized_skip_prs=frozenset(args.authorized_skip_pr), - now=now, - ) - json.dump([update.as_dict() for update in updates], sys.stdout, indent=2) - sys.stdout.write("\n") - return 0 - - token = os.environ.get(args.token_env) - if not token: - parser.error(f"{args.token_env} must contain a GitHub token") - client = GitHubClient(args.repository, token, args.api_url) - while True: - try: - updates = reconcile(client, policy, args.apply) - except RuntimeError as error: - if args.watch <= 0: - raise - print(f"::warning::{error}; retrying", file=sys.stderr) - time.sleep(args.watch) - continue - print(json.dumps([update.as_dict() for update in updates], separators=(",", ":"))) - if args.watch <= 0: - return 0 - time.sleep(args.watch) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/utils/test_authorize_skip_queue.py b/utils/test_authorize_skip_queue.py deleted file mode 100644 index bc800bb401..0000000000 --- a/utils/test_authorize_skip_queue.py +++ /dev/null @@ -1,92 +0,0 @@ -from pathlib import Path - -from authorize_skip_queue import active_label_actor, is_authorized, load_skip_policy - - -class FakeApi: - def __init__(self, events, memberships): - self.events = events - self.memberships = memberships - self.membership_requests = [] - - def paged(self, path): - assert path == "/repos/SemiAnalysisAI/InferenceX/issues/42/timeline" - return self.events - - def request(self, path): - actor = path.rsplit("/", 1)[-1] - self.membership_requests.append(actor) - return self.memberships.get(actor, {"state": "inactive"}) - - -def labeled(actor): - return { - "event": "labeled", - "label": {"name": "skip_queue"}, - "actor": {"login": actor}, - } - - -def unlabeled(actor): - return { - "event": "unlabeled", - "label": {"name": "skip_queue"}, - "actor": {"login": actor}, - } - - -def authorize(api): - return is_authorized( - api, - repository="SemiAnalysisAI/InferenceX", - pr_number=42, - organization="SemiAnalysisAI", - team_slug="core", - label_name="skip_queue", - ) - - -def test_policy_configures_skip_queue_authorization(): - policy = load_skip_policy( - Path(__file__).parents[1] / "configs" / "ci-priority.yaml" - ) - - assert policy == { - "name": "skip_queue", - "organization": "SemiAnalysisAI", - "team-slug": "core", - } - - -def test_active_label_from_active_core_member_is_authorized(): - api = FakeApi([labeled("alice")], {"alice": {"state": "active"}}) - - assert authorize(api) == (True, "alice") - - -def test_active_label_from_nonmember_is_rejected(): - api = FakeApi([labeled("mallory")], {"mallory": {"state": "inactive"}}) - - assert authorize(api) == (False, "mallory") - - -def test_latest_labeling_actor_controls_after_remove_and_readd(): - events = [labeled("alice"), unlabeled("alice"), labeled("mallory")] - api = FakeApi( - events, - { - "alice": {"state": "active"}, - "mallory": {"state": "inactive"}, - }, - ) - - assert active_label_actor(events, "skip_queue") == "mallory" - assert authorize(api) == (False, "mallory") - assert api.membership_requests == ["mallory"] - - -def test_removed_label_is_not_authorized_or_looked_up(): - api = FakeApi([labeled("alice"), unlabeled("bob")], {"alice": {"state": "active"}}) - - assert authorize(api) == (False, None) - assert api.membership_requests == [] diff --git a/utils/test_ci_priority_controller.py b/utils/test_ci_priority_controller.py deleted file mode 100644 index 91c6a36dd2..0000000000 --- a/utils/test_ci_priority_controller.py +++ /dev/null @@ -1,301 +0,0 @@ -from dataclasses import replace -from datetime import datetime, timedelta, timezone -from decimal import Decimal - -import pytest - -from ci_priority_controller import ( - GitHubClient, - QueuedJob, - Runner, - authorize_skip_requests, - parse_timestamp, - plan_label_updates, -) - - -NOW = datetime(2026, 7, 14, 18, 0, tzinfo=timezone.utc) - - -def job_label(job_id, priority): - return f"ci-job-{priority}-{job_id:032x}" - - -def job(job_id, priority, hardware, queued_minutes=0): - return QueuedJob( - id=job_id, - run_id=100 + job_id, - labels=frozenset({ - "self-hosted", - hardware, - job_label(job_id, priority), - }), - queued_at=NOW - timedelta(minutes=queued_minutes), - name=f"job-{job_id}", - ) - - -def runner(runner_id, name, *labels, busy=False, status="online"): - return Runner( - id=runner_id, - name=name, - labels=frozenset({"self-hosted", "Linux", "X64", name, *labels}), - status=status, - busy=busy, - ) - - -def test_naive_timestamps_default_to_utc(): - now = parse_timestamp("2026-07-14T18:00:00") - - updates = plan_label_updates( - [job(1, "1.000", "h100")], - [runner(11, "h100_00", "h100")], - now=now, - ) - - assert now.tzinfo == timezone.utc - assert updates[0].assigned_job_id == 1 - - -def test_assigns_best_compatible_job_to_each_idle_runner(): - jobs = [ - job(1, "5.000", "h100", queued_minutes=30), - job(2, "0.700", "b200", queued_minutes=1), - job(3, "2.500", "h100", queued_minutes=2), - ] - runners = [ - runner(10, "b200_00", "b200"), - runner(11, "h100_00", "h100"), - ] - - updates = plan_label_updates(jobs, runners, now=NOW) - - assert [(update.runner_name, update.assigned_job_id) for update in updates] == [ - ("b200_00", 2), - ("h100_00", 1), - ] - assert job_label(2, "0.700") in updates[0].labels - assert job_label(1, "5.000") in updates[1].labels - - -def test_aging_breaks_starvation_between_nearby_priorities(): - jobs = [ - job(1, "1.000", "h100", queued_minutes=8 * 60), - job(2, "2.500", "h100", queued_minutes=1), - ] - runners = [runner(11, "h100_00", "h100")] - - updates = plan_label_updates( - jobs, - runners, - now=NOW, - aging_per_hour=Decimal("0.25"), - ) - assert updates[0].assigned_job_id == 1 - -def test_authorized_skip_queue_outranks_numeric_priority(): - skip_job = job(2, "1.000", "h100") - skip_job = replace( - skip_job, - labels=skip_job.labels | {"ci-skip-queue-pr-2124"}, - ) - jobs = [ - job(1, "1000000.000", "h100"), - skip_job, - ] - runners = [runner(11, "h100_00", "h100")] - - updates = plan_label_updates( - jobs, - runners, - authorized_skip_prs=frozenset({2124}), - now=NOW, - ) - - assert updates[0].assigned_job_id == 2 - assert job_label(2, "1.000") in updates[0].labels - assert "ci-skip-queue-pr-2124" in updates[0].labels - - -def test_unauthorized_skip_queue_remains_numeric(): - skip_job = job(2, "1.000", "h100") - skip_job = replace( - skip_job, - labels=skip_job.labels | {"ci-skip-queue-pr-2124"}, - ) - - updates = plan_label_updates( - [job(1, "2.000", "h100"), skip_job], - [runner(11, "h100_00", "h100")], - now=NOW, - ) - - assert updates[0].assigned_job_id == 1 - - - -def test_controller_verifies_skip_queue_actor(): - class FixtureClient(GitHubClient): - def __init__(self): - super().__init__("owner/repo", "unused") - - def paged(self, path, key): - assert key is None - assert path == "/repos/owner/repo/issues/2124/timeline" - return [{ - "event": "labeled", - "label": {"name": "skip_queue"}, - "actor": {"login": "alice"}, - }] - - def request(self, method, path, body=None): - assert method == "GET" - assert path == "/orgs/SemiAnalysisAI/teams/core/memberships/alice" - return {"state": "active"} - - skip_job = job(2, "1.000", "h100") - skip_job = replace( - skip_job, - labels=skip_job.labels | {"ci-skip-queue-pr-2124"}, - ) - policy = { - "labels": { - "skip-queue": { - "name": "skip_queue", - "organization": "SemiAnalysisAI", - "team-slug": "core", - } - } - } - - assert authorize_skip_requests( - FixtureClient(), - [skip_job], - policy, - ) == frozenset({2124}) - - -def test_unused_idle_runner_loses_stale_priority_label(): - runners = [ - runner( - 10, - "b200_00", - "b200", - job_label(99, "5.000"), - ) - ] - - updates = plan_label_updates([], runners, now=NOW) - - assert len(updates) == 1 - assert updates[0].assigned_job_id is None - assert all(not label.startswith("ci-job-") for label in updates[0].labels) - - -def test_busy_and_offline_runners_are_never_relabelled(): - runners = [ - runner(10, "b200_00", "b200", job_label(98, "5.000"), busy=True), - runner(11, "b200_01", "b200", job_label(99, "5.000"), status="offline"), - ] - - assert plan_label_updates([job(1, "0.700", "b200")], runners, now=NOW) == [] - - -def test_exact_runner_name_remains_a_compatibility_constraint(): - jobs = [job(1, "1.000", "h100_01")] - runners = [ - runner(10, "h100_00", "h100"), - runner(11, "h100_01", "h100"), - ] - - updates = plan_label_updates(jobs, runners, now=NOW) - - assigned = [update for update in updates if update.assigned_job_id is not None] - assert len(assigned) == 1 - assert assigned[0].runner_name == "h100_01" - - -def test_preserves_scarce_runner_for_exact_job(): - jobs = [ - job(1, "5.000", "h100"), - job(2, "4.000", "h100_00"), - ] - runners = [ - runner(10, "h100_00", "h100"), - runner(11, "h100_01", "h100"), - ] - - updates = plan_label_updates(jobs, runners, now=NOW) - - assert [(update.runner_name, update.assigned_job_id) for update in updates] == [ - ("h100_00", 2), - ("h100_01", 1), - ] - - -@pytest.mark.parametrize( - ("valid", "invalid", "message"), - [ - ( - job_label(1, "1.000"), - "ci-job--1-00000000000000000000000000000001", - "invalid CI job", - ), - ( - job_label(1, "1.000"), - "ci-job-1.000-not-a-hash", - "invalid CI job", - ), - ("ci-skip-queue-pr-2124", "ci-skip-queue-pr-zero", "invalid skip_queue"), - ], -) -def test_rejects_malformed_scheduling_labels(valid, invalid, message): - queued_job = job(1, "1.000", "h100") - queued_job = replace( - queued_job, - labels=(queued_job.labels - {valid}) | {invalid}, - ) - - with pytest.raises(ValueError, match=message): - plan_label_updates( - [queued_job], - [runner(11, "h100_00", "h100")], - now=NOW, - ) - - -def test_discovers_queued_jobs_inside_in_progress_workflow_runs(): - class FixtureClient(GitHubClient): - def __init__(self): - super().__init__("owner/repo", "unused") - self.paths = [] - - def paged(self, path, key): - self.paths.append(path) - if "status=queued" in path: - return [{"id": 1}] - if "status=in_progress" in path: - return [{"id": 2}] - if "/runs/1/jobs" in path: - return [{"id": 11, "status": "completed", "labels": []}] - if "/runs/2/jobs" in path: - return [{ - "id": 22, - "status": "queued", - "created_at": "2026-07-14T18:00:00Z", - "labels": [ - "self-hosted", - "b200", - job_label(22, "1.000"), - ], - }] - raise AssertionError(path) - - client = FixtureClient() - - jobs = client.queued_jobs() - - assert [queued_job.id for queued_job in jobs] == [22] - assert any("status=queued" in path for path in client.paths) - assert any("status=in_progress" in path for path in client.paths) From e8a641d0609e188ed590489a51b1d1ded20845c2 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:19:04 -0500 Subject: [PATCH 16/21] Add priority scheduler canary workflow --- .../workflows/priority-scheduler-canary.yml | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/priority-scheduler-canary.yml diff --git a/.github/workflows/priority-scheduler-canary.yml b/.github/workflows/priority-scheduler-canary.yml new file mode 100644 index 0000000000..aed54a9ac1 --- /dev/null +++ b/.github/workflows/priority-scheduler-canary.yml @@ -0,0 +1,79 @@ +name: Priority Scheduler Canary +run-name: Priority Scheduler Canary - ${{ inputs.runner }} at p${{ inputs.score }} + +on: + workflow_dispatch: + inputs: + runner: + description: "Runner label to test" + required: true + type: choice + default: b200 + options: + - b200 + - b300 + - h100 + - h200 + - mi355x + score: + description: "Numeric priority score" + required: true + type: string + default: "1.000" + +permissions: + contents: read + +jobs: + prepare: + name: Prepare one-shot scheduling label + runs-on: ubuntu-latest + outputs: + runs-on: ${{ steps.labels.outputs.runs-on }} + job-label: ${{ steps.labels.outputs.job-label }} + steps: + - id: labels + name: Generate unique canary label + shell: python + env: + RUNNER_LABEL: ${{ inputs.runner }} + PRIORITY_SCORE: ${{ inputs.score }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + run: | + import hashlib + import json + import os + import re + + score = os.environ["PRIORITY_SCORE"] + if re.fullmatch(r"[0-9]+(?:\.[0-9]+)?", score) is None: + raise SystemExit(f"Invalid numeric priority: {score}") + + material = f"{os.environ['RUN_ID']}:{os.environ['RUN_ATTEMPT']}:canary" + token = hashlib.sha256(material.encode()).hexdigest()[:32] + job_label = f"ci-job-{score}-{token}" + runs_on = ["self-hosted", os.environ["RUNNER_LABEL"], job_label] + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"job-label={job_label}\n") + output.write(f"runs-on={json.dumps(runs_on, separators=(',', ':'))}\n") + + canary: + name: Verify priority dispatch + needs: prepare + runs-on: ${{ fromJSON(needs.prepare.outputs.runs-on) }} + timeout-minutes: 10 + steps: + - name: Confirm assignment + env: + JOB_LABEL: ${{ needs.prepare.outputs.job-label }} + RUNNER_LABEL: ${{ inputs.runner }} + run: | + echo "Controller dispatched $JOB_LABEL to $RUNNER_LABEL runner $RUNNER_NAME" + { + echo "### Priority scheduler canary passed" + echo "- Job label: \`$JOB_LABEL\`" + echo "- Runner class: \`$RUNNER_LABEL\`" + echo "- Runner: \`$RUNNER_NAME\`" + } >> "$GITHUB_STEP_SUMMARY" From f633830982d213d1190c0c15a788905e5ae11438 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:55:37 -0500 Subject: [PATCH 17/21] Harden CI priority scheduling contracts --- .../workflows/benchmark-multinode-tmpl.yml | 20 +++---- .github/workflows/benchmark-tmpl.yml | 20 +++---- .github/workflows/collectivex-sweep.yml | 17 ++++-- .github/workflows/e2e-tests.yml | 37 ++++++++---- .../workflows/priority-scheduler-canary.yml | 60 ++++++++++++++++--- .github/workflows/profile.yml | 31 +++++++--- .github/workflows/run-sweep.yml | 23 +++++-- .github/workflows/speedbench-al.yml | 17 ++++-- .github/workflows/test-changelog-gate.yml | 9 +++ configs/ci-priority.yaml | 3 +- .../test_run_sweep_gating.py | 55 +++++++++++++++-- utils/ci_priority.py | 44 +++++++------- utils/test_ci_priority.py | 49 ++++++++++++++- 13 files changed, 293 insertions(+), 92 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 275dccaaa0..985065df99 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -250,20 +250,20 @@ jobs: ( inputs.skip-queue-pr != '' && format( - '["self-hosted","{0}","ci-job-{1}-{2}","ci-skip-queue-pr-{3}"]', - inputs.runner, - inputs.priority, - inputs.queue-token, - inputs.skip-queue-pr + '["self-hosted",{0},{1},{2},{3}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)), + toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) ) || format( - '["self-hosted","{0}","ci-job-{1}-{2}"]', - inputs.runner, - inputs.priority, - inputs.queue-token + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)) ) ) || - format('["{0}"]', inputs.runner) + format('[{0}]', toJSON(inputs.runner)) ) }} timeout-minutes: 480 name: >- diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index e9e31f9d96..f3cef5413f 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -180,20 +180,20 @@ jobs: ( inputs.skip-queue-pr != '' && format( - '["self-hosted","{0}","ci-job-{1}-{2}","ci-skip-queue-pr-{3}"]', - inputs.runner, - inputs.priority, - inputs.queue-token, - inputs.skip-queue-pr + '["self-hosted",{0},{1},{2},{3}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)), + toJSON(format('ci-skip-queue-pr-{0}', inputs.skip-queue-pr)) ) || format( - '["self-hosted","{0}","ci-job-{1}-{2}"]', - inputs.runner, - inputs.priority, - inputs.queue-token + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)), + toJSON(format('ci-attempt-{0}', github.run_attempt)) ) ) || - format('["{0}"]', inputs.runner) + format('[{0}]', toJSON(inputs.runner)) ) }} timeout-minutes: 500 name: >- diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml index b326e9cdc1..0d685e0506 100644 --- a/.github/workflows/collectivex-sweep.yml +++ b/.github/workflows/collectivex-sweep.yml @@ -51,6 +51,7 @@ jobs: INPUT_EXCLUDE_SKUS: ${{ inputs.exclude_skus }} INPUT_EP_SIZES: ${{ inputs.ep_sizes }} RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} run: | set -euo pipefail args=(--backend "$INPUT_BACKEND") @@ -69,7 +70,7 @@ jobs: cells = matrix["include"] for index, cell in enumerate(cells): canonical = json.dumps(cell, sort_keys=True, separators=(",", ":")) - material = f"{os.environ['RUN_ID']}:{index}:{canonical}".encode() + material = f"{os.environ['RUN_ID']}:{os.environ['RUN_ATTEMPT']}:{index}:{canonical}".encode() cell["queue-token"] = hashlib.sha256(material).hexdigest()[:32] slim = {"include": cells} with open(sys.argv[1], "a", encoding="utf-8") as output: @@ -100,12 +101,16 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( - '["self-hosted","{0}","ci-job-{1}-{2}"]', - matrix.sku, - needs.setup.outputs.priority, - matrix.queue-token + '["self-hosted",{0},{1},{2}]', + toJSON(matrix.sku), + toJSON(format( + 'ci-job-{0}-{1}', + needs.setup.outputs.priority, + matrix.queue-token + )), + toJSON(format('ci-attempt-{0}', github.run_attempt)) ) || - format('["{0}"]', matrix.sku) + format('[{0}]', toJSON(matrix.sku)) ) }} name: p${{ needs.setup.outputs.priority }} | ${{ matrix.sku }} ${{ matrix.backend }} shard ${{ matrix.id }} timeout-minutes: 350 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 06d9154ddb..9e941df5e9 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -63,6 +63,14 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.sha }} + - name: Checkout priority scheduler tooling + if: ${{ inputs.ref && inputs.ref != '' }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.workflow_sha }} + path: .ci-priority + persist-credentials: false + - id: get-jobs env: @@ -71,17 +79,24 @@ jobs: pip install pydantic CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py \ ${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}) - CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ - "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ - --event-name "${{ github.event_name }}" \ - --queue-namespace "${{ github.run_id }}" \ - --labels-json "$PR_LABELS") - AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x]))") - MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))") - SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))") - MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))") - EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))") - MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))") + PRIORITY_ROOT="${GITHUB_WORKSPACE}" + if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then + PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority" + fi + score_matrix() { + local family="$1" + python3 "${PRIORITY_ROOT}/utils/ci_priority.py" \ + --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ + --event-name "${{ github.event_name }}" \ + --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}:${family}" \ + --labels-json "$PR_LABELS" + } + AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x]))" | score_matrix agentic) + MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))" | score_matrix multi-agentic) + SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) + MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) + EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) + MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-eval) echo "agentic-config=$AGENTIC" >> $GITHUB_OUTPUT echo "multi-node-agentic-config=$MULTI_AGENTIC" >> $GITHUB_OUTPUT echo "single-node-config=$SINGLE" >> $GITHUB_OUTPUT diff --git a/.github/workflows/priority-scheduler-canary.yml b/.github/workflows/priority-scheduler-canary.yml index aed54a9ac1..5a62b8b980 100644 --- a/.github/workflows/priority-scheduler-canary.yml +++ b/.github/workflows/priority-scheduler-canary.yml @@ -23,14 +23,14 @@ on: permissions: contents: read + actions: write jobs: prepare: name: Prepare one-shot scheduling label runs-on: ubuntu-latest outputs: - runs-on: ${{ steps.labels.outputs.runs-on }} - job-label: ${{ steps.labels.outputs.job-label }} + job-label-base: ${{ steps.labels.outputs.job-label-base }} steps: - id: labels name: Generate unique canary label @@ -52,28 +52,70 @@ jobs: material = f"{os.environ['RUN_ID']}:{os.environ['RUN_ATTEMPT']}:canary" token = hashlib.sha256(material.encode()).hexdigest()[:32] - job_label = f"ci-job-{score}-{token}" - runs_on = ["self-hosted", os.environ["RUNNER_LABEL"], job_label] + job_label_base = f"ci-job-{score}-{token}" with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: - output.write(f"job-label={job_label}\n") - output.write(f"runs-on={json.dumps(runs_on, separators=(',', ':'))}\n") + output.write(f"job-label-base={job_label_base}\n") canary: name: Verify priority dispatch needs: prepare - runs-on: ${{ fromJSON(needs.prepare.outputs.runs-on) }} + runs-on: >- + ${{ fromJSON( + github.run_attempt == 1 && + format( + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(needs.prepare.outputs.job-label-base), + toJSON(format('ci-attempt-{0}', github.run_attempt)) + ) || + '["ubuntu-latest"]' + ) }} timeout-minutes: 10 steps: + - name: Reject partial rerun + if: github.run_attempt != 1 + run: | + echo "Priority canaries are one-shot; dispatch a fresh workflow run." >&2 + exit 1 - name: Confirm assignment + if: github.run_attempt == 1 env: - JOB_LABEL: ${{ needs.prepare.outputs.job-label }} + JOB_LABEL: ${{ needs.prepare.outputs.job-label-base }} + ATTEMPT_LABEL: ${{ format('ci-attempt-{0}', github.run_attempt) }} RUNNER_LABEL: ${{ inputs.runner }} run: | - echo "Controller dispatched $JOB_LABEL to $RUNNER_LABEL runner $RUNNER_NAME" + echo "Controller dispatched $JOB_LABEL $ATTEMPT_LABEL to $RUNNER_LABEL runner $RUNNER_NAME" { echo "### Priority scheduler canary passed" echo "- Job label: \`$JOB_LABEL\`" + echo "- Attempt label: \`$ATTEMPT_LABEL\`" echo "- Runner class: \`$RUNNER_LABEL\`" echo "- Runner: \`$RUNNER_NAME\`" } >> "$GITHUB_STEP_SUMMARY" + + watchdog: + name: Enforce canary queue deadline + needs: prepare + runs-on: ubuntu-latest + timeout-minutes: 12 + steps: + - name: Cancel if the canary is still queued after ten minutes + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + for _ in {1..40}; do + status=$(gh api \ + "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?filter=latest" \ + --jq '.jobs[] | select(.name == "Verify priority dispatch") | .status') + if [ "$status" = "in_progress" ] || [ "$status" = "completed" ]; then + exit 0 + fi + sleep 15 + done + + echo "Priority canary remained queued for ten minutes; cancelling the run." >&2 + gh api --method POST \ + "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/cancel" + exit 1 diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml index b326af5e7e..20d5895046 100644 --- a/.github/workflows/profile.yml +++ b/.github/workflows/profile.yml @@ -49,6 +49,14 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ inputs.ref || github.sha }} + - name: Checkout priority scheduler tooling + if: ${{ inputs.ref && inputs.ref != '' }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.workflow_sha }} + path: .ci-priority + persist-credentials: false + - id: gen name: Generate matrix via script @@ -56,10 +64,15 @@ jobs: pip install pydantic CLI_ARGS="test-config --config-files ${{ inputs.config-file }} --config-keys ${{ inputs.config-key }} --conc ${{ inputs.conc }}" CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py $CLI_ARGS) + PRIORITY_ROOT="${GITHUB_WORKSPACE}" + if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then + PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority" + fi CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ - "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ + "${PRIORITY_ROOT}/utils/ci_priority.py" \ + --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ --event-name "${{ github.event_name }}" \ - --queue-namespace "${{ github.run_id }}" \ + --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}" \ --labels-json '${{ inputs.moe-debug && '["ci-patchwork"]' || '[]' }}') echo "raw=$CONFIG_JSON" >> $GITHUB_OUTPUT @@ -110,12 +123,16 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( - '["self-hosted","{0}","ci-job-{1}-{2}"]', - matrix.config.runner, - matrix.config.priority, - matrix.config['queue-token'] + '["self-hosted",{0},{1},{2}]', + toJSON(matrix.config.runner), + toJSON(format( + 'ci-job-{0}-{1}', + matrix.config.priority, + matrix.config['queue-token'] + )), + toJSON(format('ci-attempt-{0}', github.run_attempt)) ) || - format('["{0}"]', matrix.config.runner) + format('[{0}]', toJSON(matrix.config.runner)) ) }} env: EXP_NAME: ${{ matrix.config.exp-name }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 8f0635167d..e8933634ce 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -14,6 +14,10 @@ concurrency: github.event.label.name != 'all-evals' && github.event.label.name != 'evals-only' && github.event.label.name != 'skip_queue' && + github.event.label.name != 'ci-patchwork' && + github.event.label.name != 'engine-patch' && + github.event.label.name != 'ci-patchwork-waived' && + github.event.label.name != 'ci-checklist-complete' && github.run_id || 'active' }} @@ -51,7 +55,11 @@ jobs: github.event.label.name == 'full-sweep-fail-fast-no-canary' || github.event.label.name == 'all-evals' || github.event.label.name == 'evals-only' || - github.event.label.name == 'skip_queue' + github.event.label.name == 'skip_queue' || + github.event.label.name == 'ci-patchwork' || + github.event.label.name == 'engine-patch' || + github.event.label.name == 'ci-patchwork-waived' || + github.event.label.name == 'ci-checklist-complete' ) outputs: skip-pr-sweep: ${{ steps.sweep_policy.outputs.skip-pr-sweep }} @@ -160,6 +168,7 @@ jobs: if: >- always() && github.event_name == 'pull_request' && + vars.PRIORITY_SCHEDULER_ENABLED == 'true' && !github.event.pull_request.draft && ( needs.check-changelog.result == 'success' || @@ -179,7 +188,7 @@ jobs: - name: Classify priority criteria with Fable id: fable continue-on-error: true - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@12531344451323133b0493233c759991ac61da12 # v1.0.174 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} track_progress: false @@ -189,7 +198,7 @@ jobs: --model 'claude-fable-5' --max-turns 8 --allowedTools "Read,Glob,Grep,Bash(git diff:*)" - --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-vllm","glm5","glm5.1","kimik2.5","kimik2.6","kimik2.7","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' + --json-schema '{"type":"object","properties":{"criteria":{"type":"array","items":{"type":"string","enum":["multi-node","agentic","eval-only","fp4","mtp","eagle","eagle3","sglang","vllm","dynamo-sglang","dynamo-vllm","glm5","glm5.1","kimik2.5","dsv4","minimaxm3","qwen3.5","dsr1","checklist-complete","patchwork"]},"uniqueItems":true},"reason":{"type":"string"}},"required":["criteria","reason"]}' prompt: | Inspect this PR's diff against its base branch. @@ -272,7 +281,11 @@ jobs: github.event.label.name == 'full-sweep-fail-fast-no-canary' || github.event.label.name == 'all-evals' || github.event.label.name == 'evals-only' || - github.event.label.name == 'skip_queue' + github.event.label.name == 'skip_queue' || + github.event.label.name == 'ci-patchwork' || + github.event.label.name == 'engine-patch' || + github.event.label.name == 'ci-patchwork-waived' || + github.event.label.name == 'ci-checklist-complete' ) ) || ( @@ -344,7 +357,7 @@ jobs: CONFIG_JSON=$(printf '%s' "$CONFIG_JSON" | python3 \ "${GITHUB_WORKSPACE}/utils/ci_priority.py" \ --event-name "${{ github.event_name }}" \ - --queue-namespace "${{ github.run_id }}" \ + --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}" \ --labels-json "$PR_LABELS" \ --pr-number "${{ github.event.pull_request.number || 0 }}" \ --criteria-json "$FABLE_CRITERIA") diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index bc9f01fab5..8a785f4b96 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -127,7 +127,8 @@ jobs: "spec-decoding": "mtp" }]') scored=$(printf '%s' "$entry" | - python3 utils/ci_priority.py --queue-namespace "${{ github.run_id }}") + python3 utils/ci_priority.py \ + --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}") echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" echo "queue-token=$(jq -r '.[0][\"queue-token\"]' <<<"$scored")" >> "$GITHUB_OUTPUT" @@ -137,12 +138,16 @@ jobs: ${{ fromJSON( vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format( - '["self-hosted","{0}","ci-job-{1}-{2}"]', - inputs.runner, - needs.setup.outputs.priority, - needs.setup.outputs.queue-token + '["self-hosted",{0},{1},{2}]', + toJSON(inputs.runner), + toJSON(format( + 'ci-job-{0}-{1}', + needs.setup.outputs.priority, + needs.setup.outputs.queue-token + )), + toJSON(format('ci-attempt-{0}', github.run_attempt)) ) || - format('["{0}"]', inputs.runner) + format('[{0}]', toJSON(inputs.runner)) ) }} timeout-minutes: 600 name: "p${{ needs.setup.outputs.priority }} | SpeedBench AL matrix | ${{ inputs.category }} | mtp=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index fa8af73919..30812246f2 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -7,12 +7,20 @@ on: - main paths: - ".claude/commands/recover-failed-ingest.md" + - ".github/workflows/benchmark-tmpl.yml" - ".github/workflows/benchmark-multinode-tmpl.yml" + - ".github/workflows/collectivex-sweep.yml" + - ".github/workflows/priority-scheduler-canary.yml" + - ".github/workflows/profile.yml" - ".github/workflows/e2e-tests.yml" - ".github/workflows/run-sweep.yml" + - ".github/workflows/speedbench-al.yml" - ".github/workflows/test-changelog-gate.yml" - "benchmarks/benchmark_lib.sh" + - "configs/ci-priority.yaml" - "benchmarks/multi_node/amd_utils/job.slurm" + - "utils/ci_priority.py" + - "utils/test_ci_priority.py" - "utils/find_reusable_sweep_run.py" - "utils/test_find_reusable_sweep_run.py" - "utils/process_changelog.py" @@ -71,6 +79,7 @@ jobs: utils/changelog_gate_tests/test_recover_failed_ingest.py \ utils/test_validate_reusable_sweep_artifacts.py \ utils/changelog_gate_tests/test_run_sweep_gating.py \ + utils/test_ci_priority.py \ utils/test_process_changelog.py \ utils/test_collect_eval_results.py \ utils/evals/test_batched_eval.py \ diff --git a/configs/ci-priority.yaml b/configs/ci-priority.yaml index f72073f0c9..f86402c23c 100644 --- a/configs/ci-priority.yaml +++ b/configs/ci-priority.yaml @@ -1,7 +1,8 @@ version: 1 # Higher scores run first. Scores are uncapped and remain human-readable in -# workflow job labels (for example, ci-job-2.750-). +# `ci-job--` workflow labels; `ci-attempt-` prevents +# stale runner labels from matching partial reruns. base-score: 1.0 adjustments: diff --git a/utils/changelog_gate_tests/test_run_sweep_gating.py b/utils/changelog_gate_tests/test_run_sweep_gating.py index ecc3a5634d..7c47f8e0f1 100644 --- a/utils/changelog_gate_tests/test_run_sweep_gating.py +++ b/utils/changelog_gate_tests/test_run_sweep_gating.py @@ -39,8 +39,14 @@ "full-sweep-fail-fast", "full-sweep-fail-fast-no-canary", } -MODIFIER_LABELS = {"all-evals", "evals-only"} -RELEVANT_LABELS = SWEEP_LABELS | MODIFIER_LABELS +MODIFIER_LABELS = {"all-evals", "evals-only", "skip_queue"} +POLICY_LABELS = { + "ci-patchwork", + "engine-patch", + "ci-patchwork-waived", + "ci-checklist-complete", +} +RELEVANT_LABELS = SWEEP_LABELS | MODIFIER_LABELS | POLICY_LABELS REUSE_ELIGIBLE_LABELS = SWEEP_LABELS - {"sweep-enabled"} REUSE_INCOMPATIBLE_LABELS = {"evals-only"} @@ -197,6 +203,7 @@ def _ctx(sc: dict) -> dict: "github.event.pull_request.draft": sc.get("draft", False), "github.event.pull_request.labels.*.name": sc.get("labels", []), "github.event.label.name": sc.get("label_name"), + "vars.PRIORITY_SCHEDULER_ENABLED": sc.get("scheduler_enabled", "true"), "github.event.head_commit.message": sc.get("msg", ""), } @@ -289,6 +296,22 @@ def run_dag(sc: dict) -> tuple[str, str, str]: {**_PR, "action": "labeled", "label_name": "evals-only", "labels": ["full-sweep-enabled", "evals-only"]}, ("success", "skipped", "RUN")), + ("PR-labeled-skip-queue-restarts-full-sweep", + {**_PR, "action": "labeled", "label_name": "skip_queue", + "labels": ["full-sweep-enabled", "skip_queue"]}, + ("success", "skipped", "RUN")), + ("PR-unlabeled-skip-queue-restarts-numeric-sweep", + {**_PR, "action": "unlabeled", "label_name": "skip_queue", + "labels": ["full-sweep-enabled"]}, + ("success", "skipped", "RUN")), + ("PR-labeled-patchwork-restarts-full-sweep", + {**_PR, "action": "labeled", "label_name": "ci-patchwork", + "labels": ["full-sweep-enabled", "ci-patchwork"]}, + ("success", "skipped", "RUN")), + ("PR-unlabeled-patchwork-restarts-full-sweep", + {**_PR, "action": "unlabeled", "label_name": "ci-patchwork", + "labels": ["full-sweep-enabled"]}, + ("success", "skipped", "RUN")), ("PR-labeled-with-unrelated-label", {**_PR, "action": "labeled", "label_name": "documentation", "labels": ["full-sweep-enabled"]}, ("skipped", "skipped", "SKIP")), @@ -350,6 +373,21 @@ def test_trigger_types_enable_gated_events() -> None: assert {"opened", "reopened"}.isdisjoint(PR_TYPES) +def test_priority_classifier_only_runs_when_scheduler_is_enabled() -> None: + scenario = { + **_PR, + "action": "synchronize", + "labels": ["full-sweep-enabled"], + } + disabled = _ctx({**scenario, "scheduler_enabled": "false"}) + enabled = _ctx({**scenario, "scheduler_enabled": "true"}) + disabled["needs.check-changelog.result"] = "success" + enabled["needs.check-changelog.result"] = "success" + + assert not _eval(CLASSIFY_IF, disabled) + assert _eval(CLASSIFY_IF, enabled) + + # -------------------------------------------------------------------------- # Independent reference spec of the INTENDED gating, plus an exhaustive # cross-product cross-check: every combination of the input axes is fed to @@ -428,6 +466,8 @@ def _all_scenarios() -> list[dict]: ["full-sweep-enabled", "evals-only"], ["sweep-enabled", "all-evals", "evals-only"], ["full-sweep-enabled", "all-evals", "evals-only"], + ["skip_queue"], + ["full-sweep-enabled", "skip_queue"], ] pr_axes = itertools.product( ["ready_for_review", "synchronize", "labeled", "unlabeled"], # action @@ -438,6 +478,11 @@ def _all_scenarios() -> list[dict]: "sweep-enabled", "all-evals", "evals-only", + "skip_queue", + "ci-patchwork", + "engine-patch", + "ci-patchwork-waived", + "ci-checklist-complete", "documentation", None, ], # label.name @@ -466,9 +511,9 @@ def test_exhaustive_cross_product() -> None: ] assert not mismatches, mismatches[:10] # Sanity: confirm the sweep actually covered the whole input space - # (4 actions x 2 draft x 18 label-configs x 6 label-names x 2 reuse x - # 2 changelog outcomes x 2 messages = 6912 PR cases, plus 2 push cases). - assert len(scenarios) == 6914 + # (4 actions x 2 draft x 20 label-configs x 11 label-names x 2 reuse x + # 2 changelog outcomes x 2 messages = 14080 PR cases, plus 2 push cases). + assert len(scenarios) == 14082 def test_named_cases_match_reference_spec() -> None: diff --git a/utils/ci_priority.py b/utils/ci_priority.py index 39501bc885..89b51d3ed5 100755 --- a/utils/ci_priority.py +++ b/utils/ci_priority.py @@ -45,33 +45,37 @@ def _first_prefix_adjustment(value: str, adjustments: dict[str, Any]) -> Decimal return Decimal(0) +def supported_criteria(policy: dict[str, Any]) -> frozenset[str]: + """Return every classifier fact understood by the configured policy.""" + adjustments = policy["adjustments"] + return frozenset( + { + "multi-node", + "agentic", + "eval-only", + "checklist-complete", + "patchwork", + } + | set(adjustments.get("precision", {})) + | set(adjustments.get("spec-decoding", {})) + | set(adjustments.get("framework-prefix", {})) + | set(adjustments.get("model-prefix", {})) + ) + + def _entry_from_criteria( criteria: frozenset[str], policy: dict[str, Any], entry: dict[str, Any], ) -> dict[str, Any]: - fixed = { - "multi-node", - "agentic", - "eval-only", - "fp4", - "mtp", - "eagle", - "eagle3", - "sglang", - "vllm", - "dynamo-vllm", - "checklist-complete", - "patchwork", - } - allowed = fixed | set(policy["adjustments"].get("model-prefix", {})) - unknown = criteria - allowed + unknown = criteria - supported_criteria(policy) if unknown: raise ValueError(f"Unknown CI priority criteria: {sorted(unknown)}") + precision = str(entry.get("precision", "")) spec_decoding = str(entry.get("spec-decoding", "")) framework = str(entry.get("framework", "")) model_prefix = str(entry.get("model-prefix", "")) - framework_criteria = ("sglang", "vllm", "dynamo-vllm") + framework_criteria = tuple(policy["adjustments"].get("framework-prefix", {})) model_criteria = tuple(policy["adjustments"].get("model-prefix", {})) return { "prefill": ( @@ -83,9 +87,7 @@ def _entry_from_criteria( else "" ), "eval-only": "eval-only" in criteria and entry.get("eval-only") is True, - "precision": ( - "fp4" if "fp4" in criteria and entry.get("precision") == "fp4" else "" - ), + "precision": precision if precision in criteria else "", "spec-decoding": spec_decoding if spec_decoding in criteria else "", "framework": ( framework @@ -160,7 +162,7 @@ def calculate_priority( checklist = policy["labels"].get("checklist-complete", {}) if ( (criteria is not None and "checklist-complete" in criteria) - or (criteria is None and context.labels & set(checklist.get("names", []))) + or context.labels & set(checklist.get("names", [])) ): score += _decimal(checklist.get("adjustment", 0)) diff --git a/utils/test_ci_priority.py b/utils/test_ci_priority.py index 4556efc96e..47ec74efbb 100644 --- a/utils/test_ci_priority.py +++ b/utils/test_ci_priority.py @@ -1,10 +1,20 @@ +import json +import shlex from copy import deepcopy from decimal import Decimal from pathlib import Path import pytest +import yaml -from ci_priority import PriorityContext, annotate_jobs, calculate_priority, load_policy +from ci_priority import ( + PriorityContext, + annotate_jobs, + calculate_priority, + load_policy, + queue_token, + supported_criteria, +) POLICY = load_policy(Path(__file__).parents[1] / "configs" / "ci-priority.yaml") @@ -130,6 +140,19 @@ def test_fable_criteria_drive_all_configured_adjustments(): ) == Decimal("1.000") +def test_checklist_label_applies_alongside_classifier_criteria(): + entry = {"runner": "h100", "framework": "trt"} + + assert calculate_priority( + entry, + POLICY, + PriorityContext( + labels=frozenset({"ci-checklist-complete"}), + criteria=frozenset(), + ), + ) == Decimal("1.250") + + def test_fable_criteria_reject_unknown_values_and_allow_mixed_jobs(): entry = {"runner": "h100", "framework": "vllm"} @@ -182,3 +205,27 @@ def test_annotation_only_touches_runnable_matrix_entries(): assert "priority" not in annotated["changelog_metadata"] assert "priority" not in payload["single_node"]["1k1k"][0] assert "queue-token" not in payload["single_node"]["1k1k"][0] + + +def test_classifier_schema_matches_the_policy_vocabulary(): + workflow = yaml.safe_load( + ( + Path(__file__).parents[1] / ".github" / "workflows" / "run-sweep.yml" + ).read_text() + ) + classifier = workflow["jobs"]["classify-priority"]["steps"][1] + arguments = shlex.split(classifier["with"]["claude_args"]) + schema = json.loads(arguments[arguments.index("--json-schema") + 1]) + schema_criteria = schema["properties"]["criteria"]["items"]["enum"] + + assert set(schema_criteria) == set(supported_criteria(POLICY)) + + +def test_queue_tokens_change_between_run_attempts(): + entry = {"runner": "b200", "framework": "sglang"} + + assert queue_token(entry, "123:1", ("0",)) != queue_token( + entry, + "123:2", + ("0",), + ) From 0ee9c583b9520db03a325c8e4c02b64fa8f18d28 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 22 Jul 2026 02:55:07 -0500 Subject: [PATCH 18/21] Fix priority metadata for agentic eval jobs --- .github/workflows/e2e-tests.yml | 2 ++ .github/workflows/run-sweep.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 54668fd2a8..0a077d582f 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -287,6 +287,8 @@ jobs: with: exp-name: ${{ matrix.config.exp-name }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index a58bd0842c..c6da8f7b92 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -777,6 +777,9 @@ jobs: with: exp-name: ${{ matrix.config.exp-name }} runner: ${{ matrix.config.runner }} + priority: ${{ matrix.config.priority }} + queue-token: ${{ matrix.config['queue-token'] }} + skip-queue-pr: ${{ matrix.config['skip-queue-pr'] || '' }} image: ${{ matrix.config.image }} model: ${{ matrix.config.model }} model-prefix: ${{ matrix.config.model-prefix }} From 55c9cc71bb583c7550af2c4a844e6f3ca3f718be Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:19:58 -0500 Subject: [PATCH 19/21] Harden reusable benchmark input contracts --- .github/workflows/benchmark-tmpl.yml | 1 - .github/workflows/e2e-tests.yml | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 0e32953e8a..84b778ec30 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -79,7 +79,6 @@ on: run-eval: type: boolean required: true - default: false eval-only: description: "Run only evals (skip throughput benchmark)" type: boolean diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 0a077d582f..a8b8046370 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -269,7 +269,7 @@ jobs: osl: '0' max-model-len: '0' spec-decoding: ${{ matrix.config.spec-decoding }} - disagg: 'false' + disagg: ${{ 'false' }} run-eval: false scenario-type: agentic-coding ref: ${{ inputs.ref }} @@ -306,7 +306,7 @@ jobs: osl: '0' max-model-len: '0' spec-decoding: 'none' - disagg: 'false' + disagg: ${{ 'false' }} run-eval: true eval-only: true eval-limit: ${{ inputs.eval-limit }} From 672d53a4026b657396992e8645610a0dac4d31eb Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:37:35 -0500 Subject: [PATCH 20/21] Tighten priority classification gate --- .github/workflows/run-sweep.yml | 24 +++++++++++++++++-- .../test_run_sweep_gating.py | 23 +++++++++++++++--- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index c6da8f7b92..cf956608b7 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -170,9 +170,29 @@ jobs: github.event_name == 'pull_request' && vars.PRIORITY_SCHEDULER_ENABLED == 'true' && !github.event.pull_request.draft && + needs.check-changelog.result == 'success' && + needs.check-changelog.outputs.skip-pr-sweep != 'true' && ( - needs.check-changelog.result == 'success' || - needs.check-changelog.result == 'skipped' + contains(github.event.pull_request.labels.*.name, 'sweep-enabled') || + contains(github.event.pull_request.labels.*.name, 'full-sweep-enabled') || + contains(github.event.pull_request.labels.*.name, 'non-canary-full-sweep-enabled') || + contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast') || + contains(github.event.pull_request.labels.*.name, 'full-sweep-fail-fast-no-canary') + ) && + ( + (github.event.action != 'labeled' && github.event.action != 'unlabeled') || + github.event.label.name == 'sweep-enabled' || + github.event.label.name == 'full-sweep-enabled' || + github.event.label.name == 'non-canary-full-sweep-enabled' || + github.event.label.name == 'full-sweep-fail-fast' || + github.event.label.name == 'full-sweep-fail-fast-no-canary' || + github.event.label.name == 'all-evals' || + github.event.label.name == 'evals-only' || + github.event.label.name == 'skip_queue' || + github.event.label.name == 'ci-patchwork' || + github.event.label.name == 'engine-patch' || + github.event.label.name == 'ci-patchwork-waived' || + github.event.label.name == 'ci-checklist-complete' ) permissions: contents: read diff --git a/utils/changelog_gate_tests/test_run_sweep_gating.py b/utils/changelog_gate_tests/test_run_sweep_gating.py index a7fdf9732b..9db802ebfa 100644 --- a/utils/changelog_gate_tests/test_run_sweep_gating.py +++ b/utils/changelog_gate_tests/test_run_sweep_gating.py @@ -373,7 +373,7 @@ def test_trigger_types_enable_gated_events() -> None: assert {"opened", "reopened"}.isdisjoint(PR_TYPES) -def test_priority_classifier_only_runs_when_scheduler_is_enabled() -> None: +def test_priority_classifier_matches_sweep_eligibility() -> None: scenario = { **_PR, "action": "synchronize", @@ -381,12 +381,29 @@ def test_priority_classifier_only_runs_when_scheduler_is_enabled() -> None: } disabled = _ctx({**scenario, "scheduler_enabled": "false"}) enabled = _ctx({**scenario, "scheduler_enabled": "true"}) - disabled["needs.check-changelog.result"] = "success" - enabled["needs.check-changelog.result"] = "success" + for ctx in (disabled, enabled): + ctx["needs.check-changelog.result"] = "success" + ctx["needs.check-changelog.outputs.skip-pr-sweep"] = "false" assert not _eval(CLASSIFY_IF, disabled) assert _eval(CLASSIFY_IF, enabled) + skipped_sweep = _ctx({**scenario, "msg": "fix: defer [skip-sweep]"}) + skipped_sweep["needs.check-changelog.result"] = "success" + skipped_sweep["needs.check-changelog.outputs.skip-pr-sweep"] = "true" + assert not _eval(CLASSIFY_IF, skipped_sweep) + + unrelated_label = _ctx( + { + **scenario, + "action": "labeled", + "label_name": "documentation", + } + ) + unrelated_label["needs.check-changelog.result"] = "skipped" + unrelated_label["needs.check-changelog.outputs.skip-pr-sweep"] = "false" + assert not _eval(CLASSIFY_IF, unrelated_label) + def test_reuse_dispatches_source_directly_without_artifact_relay() -> None: jobs = _WF["jobs"] From 8f2be4e55ff58946dfe2c0e380bfbb78f01ab94a Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 22 Jul 2026 03:38:22 -0500 Subject: [PATCH 21/21] Fix speedbench priority queue token --- .github/workflows/speedbench-al.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/speedbench-al.yml b/.github/workflows/speedbench-al.yml index 8a785f4b96..bc407dc435 100644 --- a/.github/workflows/speedbench-al.yml +++ b/.github/workflows/speedbench-al.yml @@ -130,7 +130,7 @@ jobs: python3 utils/ci_priority.py \ --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}") echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" - echo "queue-token=$(jq -r '.[0][\"queue-token\"]' <<<"$scored")" >> "$GITHUB_OUTPUT" + echo "queue-token=$(jq -r '.[0]["queue-token"]' <<<"$scored")" >> "$GITHUB_OUTPUT" collect-al: needs: setup