diff --git a/.github/review-metrics.yml b/.github/review-metrics.yml new file mode 100644 index 0000000000..4fdb4fea8f --- /dev/null +++ b/.github/review-metrics.yml @@ -0,0 +1,61 @@ +# Weekly review operating metrics (github/awesome-copilot#4184, phase 3). +# +# Read by eng/review-metrics.mjs, which .github/workflows/review-metrics.yml +# runs every Monday. See docs/maintainers/canvas-evidence-and-metrics.md. + +# Days of activity summarized in each report. +window_days: 7 + +# Review targets, in business days (weekends are skipped). +targets_business_days: [2, 4] + +# Base branch for contribution PRs. +base_branch: main + +# State labels (maintained by the submission gate). Open PRs without any of +# these are reported as "unlabeled". +state_labels: + - awaiting-automation + - requires-submitter-fixes + - ready-for-review + - review-in-progress + - approved + +# Risk tier labels. Open PRs without any of these are "unclassified". +risk_labels: + - merge-risk:low + - merge-risk:medium + - merge-risk:high + +# External plugin submission issues are tracked alongside PRs. +external_plugin_label: external-plugin +external_plugin_state_labels: + - awaiting-review + - requires-submitter-fixes + - ready-for-review + - awaiting-approval + +# Review-automation workflows whose failed/errored runs count toward the +# automation failure rate. Files that do not exist yet are reported as +# "not found" and skipped, so later phases can be listed ahead of time. +automation_workflows: + - submission-gate.yml + - submission-gate-writer.yml + - review-routing.yml + - canvas-smoke-test.yml + - canvas-smoke-test-comment.yml + - pr-risk-scan.yml + - pr-risk-scan-comment.yml + - label-pr-intent.yml + - label-pr-intent-writer.yml + - check-plugin-structure.yml + - external-plugin-intake.yml + - external-plugin-quality-gates.yml + - external-plugin-pr-quality-gates.yml + - external-plugin-pr-quality-gates-writer.yml + - external-plugin-command-router.yml + +# Tracking issue that receives the weekly report. +tracking_issue: + title: Review operating metrics + label: review-metrics diff --git a/.github/workflows/canvas-smoke-test-comment.yml b/.github/workflows/canvas-smoke-test-comment.yml new file mode 100644 index 0000000000..f5dfd2f30a --- /dev/null +++ b/.github/workflows/canvas-smoke-test-comment.yml @@ -0,0 +1,157 @@ +name: Canvas Smoke Test โ€” Comment + +# Writer half of the canvas-smoke-test reader/writer split. The reader +# (canvas-smoke-test.yml) runs on untrusted PR code with a read-only +# token; this workflow only reads its artifact and never checks out PR code. + +on: + workflow_run: + workflows: ["Canvas Smoke Test"] + types: [completed] + +permissions: + actions: read + issues: write + pull-requests: write + +jobs: + comment: + runs-on: ubuntu-latest + if: github.event.workflow_run.event == 'pull_request' + steps: + - name: Download review artifact + id: download + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: canvas-smoke-test-results + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Upsert PR comment + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + with: + script: | + const fs = require('fs'); + const marker = ''; + + if (!fs.existsSync('report.md')) { + core.info('No canvas smoke test report artifact found. Skipping.'); + return; + } + + let status = 'unknown'; + try { + status = JSON.parse(fs.readFileSync('results.json', 'utf8')).status || 'unknown'; + } catch { + core.warning('results.json is missing or invalid.'); + } + + const workflowRun = context.payload.workflow_run; + const artifactPr = fs.existsSync('pr-number.txt') ? fs.readFileSync('pr-number.txt', 'utf8').trim() : ''; + const prNumber = /^[1-9][0-9]*$/.test(artifactPr) ? Number(artifactPr) : workflowRun.pull_requests?.[0]?.number; + if (!prNumber) { + core.warning('Could not determine PR number. Skipping.'); + return; + } + + // The artifact is untrusted: bind the target PR to the triggering run + // (base repo, head repo/ref/SHA, and a unique association) before writing. + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + if (pr.state !== 'open') { + core.info(`Skipping non-open PR #${prNumber}.`); + return; + } + const expectedBaseRepository = `${context.repo.owner}/${context.repo.repo}`.toLowerCase(); + const runHeadRepository = String(workflowRun.head_repository?.full_name || ''); + const runHeadRepositoryParts = runHeadRepository.split('/'); + const runHeadRef = String(workflowRun.head_branch || ''); + if (String(pr.base?.repo?.full_name || '').toLowerCase() !== expectedBaseRepository) { + core.setFailed(`PR #${prNumber} does not target this repository.`); + return; + } + if (pr.head.sha !== workflowRun.head_sha) { + core.info(`PR #${prNumber} head ${pr.head.sha} does not match run head ${workflowRun.head_sha}. Skipping stale or mismatched report.`); + return; + } + if ( + runHeadRepositoryParts.length !== 2 || + !runHeadRepositoryParts[0] || + !runHeadRepositoryParts[1] || + !runHeadRef || + String(pr.head?.repo?.full_name || '').toLowerCase() !== runHeadRepository.toLowerCase() || + String(pr.head?.ref || '') !== runHeadRef + ) { + core.setFailed(`PR #${prNumber} head repository/ref did not match workflow_run.`); + return; + } + const workflowRunPullRequests = Array.isArray(workflowRun.pull_requests) ? workflowRun.pull_requests : []; + if (workflowRunPullRequests.length > 0) { + if (!workflowRunPullRequests.some((pullRequest) => pullRequest.number === prNumber)) { + core.setFailed(`PR #${prNumber} was not present in workflow_run.pull_requests.`); + return; + } + } else { + const candidatePullRequests = await github.paginate(github.rest.pulls.list, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + head: `${runHeadRepositoryParts[0]}:${runHeadRef}`, + per_page: 100, + }); + const trustedMatches = candidatePullRequests.filter((candidate) => + candidate.head?.sha === workflowRun.head_sha && + String(candidate.head?.ref || '') === runHeadRef && + String(candidate.head?.repo?.full_name || '').toLowerCase() === runHeadRepository.toLowerCase() && + String(candidate.base?.repo?.full_name || '').toLowerCase() === expectedBaseRepository + ); + if (trustedMatches.length !== 1 || trustedMatches[0].number !== prNumber) { + core.setFailed(`PR #${prNumber} could not be uniquely associated with workflow_run.`); + return; + } + } + + let body = fs.readFileSync('report.md', 'utf8').replace(/@/g, '@\u200b'); + const maxLength = 65000; + if (body.length > maxLength) { + body = `${body.slice(0, maxLength)}\n\n_...(truncated)..._`; + } + if (!body.includes(marker)) { + body = `${marker}\n${body}`; + } + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + }); + const existing = comments.find((comment) => + comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker)); + + if (status === 'skipped' && !existing) { + core.info('No canvas changes and no previous comment. Nothing to post.'); + return; + } + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + core.info(`Updated canvas smoke test comment ${existing.id}`); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body, + }); + core.info('Created canvas smoke test comment'); + } \ No newline at end of file diff --git a/.github/workflows/canvas-smoke-test.yml b/.github/workflows/canvas-smoke-test.yml new file mode 100644 index 0000000000..62d3daad04 --- /dev/null +++ b/.github/workflows/canvas-smoke-test.yml @@ -0,0 +1,111 @@ +name: Canvas Smoke Test + +# Reader half of the canvas-smoke-test reader/writer split. Runs untrusted PR +# content with a read-only token and no secrets; extension code is parsed and +# compiled but never executed. canvas-smoke-test-comment.yml posts the report. + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + paths: + - "extensions/**" + - "plugins/**" + - ".github/workflows/canvas-smoke-test.yml" + - "eng/canvas-smoke-test.mjs" + +permissions: + contents: read + +concurrency: + group: canvas-smoke-test-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + canvas-smoke-test: + name: canvas-smoke-test + runs-on: ubuntu-latest + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + CANVAS_PREVIEW_MIN_WIDTH: ${{ vars.CANVAS_PREVIEW_MIN_WIDTH }} + CANVAS_PREVIEW_MIN_HEIGHT: ${{ vars.CANVAS_PREVIEW_MIN_HEIGHT }} + CANVAS_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + cache: "npm" + + - name: Detect canvas targets + id: detect + run: | + set -euo pipefail + git diff --name-only "${BASE_SHA}...HEAD" > "${RUNNER_TEMP}/changed-files.txt" + node ./eng/canvas-smoke-test.mjs \ + --changed-files "${RUNNER_TEMP}/changed-files.txt" \ + --base-ref "${BASE_SHA}" \ + --detect-only + + - name: Write skipped report + if: steps.detect.outputs.canvas != 'true' + run: | + set -euo pipefail + mkdir -p canvas-smoke-results + printf '\n## ๐Ÿงฉ Canvas smoke test\n\nโญ๏ธ **Skipped** โ€” no canvas extension or extension-bearing plugin paths changed.\n' > canvas-smoke-results/report.md + printf '{"schema_version":"canvas-smoke-test/v1","status":"skipped"}\n' > canvas-smoke-results/results.json + echo "${PR_NUMBER}" > canvas-smoke-results/pr-number.txt + echo "${HEAD_SHA}" > canvas-smoke-results/head-sha.txt + cat canvas-smoke-results/report.md >> "$GITHUB_STEP_SUMMARY" + + - name: Install dependencies + if: steps.detect.outputs.canvas == 'true' + run: npm ci --ignore-scripts + + - name: Install GitHub Copilot CLI + if: steps.detect.outputs.canvas == 'true' + continue-on-error: true + run: npm install -g @github/copilot + + - name: Run canvas smoke test + if: steps.detect.outputs.canvas == 'true' + run: | + set -uo pipefail + export CANVAS_PREVIEW_BASE_URL="https://raw.githubusercontent.com/${HEAD_REPO}/${HEAD_SHA}/" + node ./eng/canvas-smoke-test.mjs \ + --changed-files "${RUNNER_TEMP}/changed-files.txt" \ + --base-ref "${BASE_SHA}" \ + --install require \ + --output-dir canvas-smoke-results > /dev/null + exit_code=$? + if [ ! -f canvas-smoke-results/report.md ]; then + mkdir -p canvas-smoke-results + printf '\n## ๐Ÿงฉ Canvas smoke test\n\nโš ๏ธ **Infrastructure error** โ€” the checker crashed. See the workflow logs.\n' > canvas-smoke-results/report.md + echo '{"schema_version":"canvas-smoke-test/v1","status":"infra_error"}' > canvas-smoke-results/results.json + fi + echo "${PR_NUMBER}" > canvas-smoke-results/pr-number.txt + echo "${HEAD_SHA}" > canvas-smoke-results/head-sha.txt + cat canvas-smoke-results/report.md >> "$GITHUB_STEP_SUMMARY" + case "$exit_code" in + 0) ;; + 1) echo "::error::Canvas smoke test found contribution issues. See the job summary." ;; + *) echo "::error::Canvas smoke test hit an infrastructure error (not a contribution failure)." ;; + esac + exit "$exit_code" + + - name: Upload review artifact + if: always() && hashFiles('canvas-smoke-results/report.md') != '' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: canvas-smoke-test-results + path: canvas-smoke-results/ + retention-days: 7 diff --git a/.github/workflows/review-metrics.yml b/.github/workflows/review-metrics.yml new file mode 100644 index 0000000000..a42512ea49 --- /dev/null +++ b/.github/workflows/review-metrics.yml @@ -0,0 +1,76 @@ +name: Review Operating Metrics + +# Publishes weekly review operating metrics (#4184) to a pinned tracking +# issue labeled `review-metrics`. Reads repository data only; the sole write is +# the tracking issue. Definitions: docs/maintainers/canvas-evidence-and-metrics.md + +on: + schedule: + # Mondays 14:00 UTC, covering the previous seven days. + - cron: "0 14 * * 1" + workflow_dispatch: + inputs: + window_days: + description: "Days of activity to summarize" + required: false + default: "7" + type: string + dry_run: + description: "Only write the job summary; do not update the tracking issue" + required: false + default: false + type: boolean + +permissions: + contents: read + +concurrency: + group: review-metrics + cancel-in-progress: false + +jobs: + metrics: + name: review-metrics + if: github.repository_owner == 'github' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + issues: write + pull-requests: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + cache: npm + + - name: Install dependencies + run: npm ci --ignore-scripts + + - name: Compute and publish metrics + env: + GITHUB_TOKEN: ${{ github.token }} + METRICS_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + WINDOW_DAYS: ${{ inputs.window_days || '7' }} + DRY_RUN: ${{ inputs.dry_run && 'true' || 'false' }} + run: | + set -euo pipefail + args=(--repo "$GITHUB_REPOSITORY" --window-days "$WINDOW_DAYS" --output-dir "$RUNNER_TEMP/review-metrics") + if [ "$DRY_RUN" = "true" ]; then args+=(--dry-run); fi + node ./eng/review-metrics.mjs "${args[@]}" + + - name: Upload metrics + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: review-metrics + path: ${{ runner.temp }}/review-metrics + if-no-files-found: ignore + retention-days: 90 diff --git a/.github/workflows/setup-labels.yml b/.github/workflows/setup-labels.yml index 8d78d5ad3a..56fcffdee1 100644 --- a/.github/workflows/setup-labels.yml +++ b/.github/workflows/setup-labels.yml @@ -90,6 +90,11 @@ jobs: color: 'FBCA04', description: 'External plugin awaiting maintainer approval' }, + // Review metrics tracking label (see docs/maintainers/canvas-evidence-and-metrics.md) + 'review-metrics': { + color: 'C5DEF5', + description: 'Weekly review operating metrics tracking issue' + }, // Submission gate labels (docs/maintainers/submission-gate.md). // ready-for-review, requires-submitter-fixes, and approved are shared with intake above. 'awaiting-automation': { diff --git a/docs/maintainers/canvas-evidence-and-metrics.md b/docs/maintainers/canvas-evidence-and-metrics.md new file mode 100644 index 0000000000..155a7a88b4 --- /dev/null +++ b/docs/maintainers/canvas-evidence-and-metrics.md @@ -0,0 +1,114 @@ +# Canvas review evidence and review metrics + +This guide covers phase 3 ("safe acceleration") of [#4184](https://github.com/github/awesome-copilot/issues/4184): + +1. [Canvas review evidence](#canvas-review-evidence): the `canvas-smoke-test` check. +2. [Weekly operating metrics](#weekly-operating-metrics): `.github/review-metrics.yml` and `.github/workflows/review-metrics.yml`. +3. [Deferred: auto-merge](#deferred-auto-merge). +4. [Portability to MOS3](#portability-to-mos3). + +Phase 1 owns reviewer routing (CODEOWNERS, `.github/review-routing.yml`, reviewer labels). Phase 2 owns the `submission-gate` check, `merge-risk:*` tiers, and state labels. This phase consumes those signals and falls back gracefully when they do not exist yet. + +## Canvas review evidence + +### What runs + +`.github/workflows/canvas-smoke-test.yml` runs on PRs to `main` that change `extensions/**`, `plugins/**`, or the checker and its workflow. It first runs the checker's target detection without installing dependencies or the Copilot CLI; ordinary plugin-only changes that do not add, remove, or reference canvas extensions skip the expensive steps. Canvas extensions are still submitted only under `extensions/`, so `validate-canvas-extensions.yml` keeps its existing scope and is unchanged. The `canvas-smoke-test` job reports: + +- **Skipped (success)** when the PR changes no canvas extension or extension-bearing plugin paths, including ordinary plugin-only changes and checker/workflow-only changes. +- **Pass (removal accepted)** when an extension directory and its plugin were both deleted. Deleting only `extension.mjs`, or leaving a plugin that still references or was the direct plugin for a deleted extension, fails. +- **Pass or fail** otherwise, from `eng/canvas-smoke-test.mjs`. + +For each affected extension, the checker: + +| Area | Checks | +|---|---| +| Module graph | Parses `extension.mjs` and every reachable local module **without executing it** (`vm.SourceTextModule` / `vm.compileFunction`). Reachability follows static imports, literal dynamic `import()` calls, and literal CommonJS `require()` calls (call-shaped text inside strings, templates, and regexes is ignored). `package.json` `imports` aliases (`#name`) resolve through the same checks; alias shapes the checker cannot analyze fail. Every reachable reference must resolve to a file inside the extension, a Node.js built-in, the host-provided `@github/copilot-sdk`, or a runtime dependency (`dependencies`, `optionalDependencies`, `peerDependencies`). Remote (`http:`/`https:`), `data:`, absolute, `..`, missing, `devDependencies`-only, and undeclared references fail. Modules not reachable from `extension.mjs` (often browser assets for the canvas webview) produce warnings instead. | +| Files | Missing referenced files, unsafe paths (`..`, absolute paths, `file:` URLs), symlinks, committed `node_modules`, native binaries (ELF, PE, Mach-O, `.node`, `.dll`, `.so`, WebAssembly), executable git modes, and files over 5 MB fail. Shell, batch, PowerShell, Python, Ruby, and Perl scripts, shebang files, and unrecognized binaries are reported as warnings for the reviewer. | +| Preview | `assets/preview.png` must exist, decode as a real PNG (signature, chunk CRCs, `IHDR`, palette, `IEND`, inflated image data and filter bytes), be at most 5 MB, decode to at most 256 MiB of image data (checked before decompression), and meet the minimum dimensions. | +| Plugin | Materializes the plugin with `eng/materialize-plugins.mjs` in a temporary copy, validates the served `plugin.json` against the Agent Plugins schema, and confirms `com.github.copilot/extensions//extension.mjs` exists. | +| Install | Installs the materialized plugin with the GitHub Copilot CLI from an ephemeral local marketplace, in an isolated `COPILOT_HOME` with all tokens removed, and verifies it with `copilot plugin list --json`. CI uses `--install require`. | +| Capabilities | Summarizes what an extension uses (for example filesystem, child processes, network, `process.env`, dynamic code) so reviewers can compare it with the stated purpose. | + +Preview minimum dimensions default to **400 ร— 160 px**. Every existing preview meets this; the smallest are 475 ร— 440, 720 ร— 165, 544 ร— 306, and 657 ร— 270. Change the minimum with the repository variables `CANVAS_PREVIEW_MIN_WIDTH` and `CANVAS_PREVIEW_MIN_HEIGHT`, or with `--min-preview-width` and `--min-preview-height` locally. + +### Review artifact + +The job writes a concise report to the job summary and uploads the `canvas-smoke-test-results` artifact (7-day retention) with `report.md`, `results.json`, and the preview images. The report includes plugin and extension metadata, changed files, capabilities, the preview, and every validation and smoke-test result. + +`.github/workflows/canvas-smoke-test-comment.yml` follows the repository's reader/writer split. The reader (`canvas-smoke-test.yml`) runs untrusted PR content with a read-only token and no secrets. The writer runs from `main` on `workflow_run` and binds the artifact to the triggering run before writing: the PR must target this repository, its head repository, branch, and SHA must match the run, and it must appear in `workflow_run.pull_requests` or be the only open PR with that head. It also neutralizes `@` mentions, and upserts one PR comment marked ``. It does not comment on skipped PRs that never had a report. + +### Running locally + +```bash +node eng/canvas-smoke-test.mjs --all --install never # every extension, no CLI install +node eng/canvas-smoke-test.mjs --changed-files changed.txt # only what a diff touches +node --test eng/canvas-smoke-test.test.mjs +``` + +Exit codes: `0` pass or skipped, `1` contribution problems, `2` infrastructure error. + +## Weekly operating metrics + +`.github/workflows/review-metrics.yml` runs every Monday at 14:00 UTC and on demand. `eng/review-metrics.mjs` collects read-only data and then: + +- writes the report to the job summary and uploads the `review-metrics` artifact (`metrics.json` and `report.md`, 90-day retention); +- updates the body of the tracking issue labeled `review-metrics`, creating and pinning it the first time; +- adds the week's report to that issue as a comment so trends stay visible. + +### Definitions + +All times are UTC. The window is the previous `window_days` (default 7) days. PRs authored by bots are excluded from contribution counts and review timings. A **maintainer review** is a submitted review from a user with write access who is not the PR author and not a bot. + +| Metric | Definition | +|---|---| +| Open contributions by state | Open, non-draft contribution PRs to `main`, grouped by phase 2 state label; PRs without one are `unlabeled`. Open `external-plugin` issues are grouped by their state labels. | +| Open contributions by risk tier | The same PRs grouped by `merge-risk:*`; PRs without one are `unclassified`. | +| Time to first review | For PRs whose first maintainer review landed in the window: first review time minus the later of creation and the last ready-for-review event. Reviews submitted before that start (for example before a PR went back to draft) are ignored. Median and p90 (linear interpolation), in wall-clock hours or days. | +| Time to merge | For PRs merged in the window: merge time minus creation time. Median and p90. | +| Reviews per maintainer | Maintainer reviews submitted in the window, and the distinct PRs each maintainer reviewed. | +| Reviewer concentration | Based on distinct PRs reviewed per maintainer. **Top-reviewer share** is the busiest maintainer's share. **HHI** (Herfindahl-Hirschman index) is the sum of squared shares ร— 10,000: 10,000 means one reviewer did everything, and above 2,500 is highly concentrated. **Effective reviewers** is 1 divided by the unscaled HHI. | +| Items past 2 and 4 business days | Items waiting on a maintainer longer than each target: open, non-draft PRs without a maintainer review since the clock started and without `requires-submitter-fixes` (the clock starts at creation or the last ready-for-review event), and `external-plugin` issues labeled `ready-for-review` or `awaiting-approval` (the clock starts when that label was last added). Business days count Monday through Friday only, with partial days counted fractionally. | +| Automation failure rate | For the configured `automation_workflows`: runs created in the window with conclusion `failure`, `timed_out`, or `startup_failure`, divided by completed runs excluding `cancelled`, `skipped`, `neutral`, `action_required`, and `stale`. Workflows that do not exist yet are listed as "not found". | + +`.github/review-metrics.yml` configures the window, targets, label sets, workflows, and tracking issue. Missing labels never cause errors. + +### Running locally + +```bash +node eng/review-metrics.mjs --repo github/awesome-copilot --dry-run --output-dir ./metrics +node --test eng/review-metrics.test.mjs +``` + +## Deferred: auto-merge + +Safe auto-merge was proposed in this phase and removed before merge. Arming GitHub auto-merge from automation would codify workarounds to repository and organization policy (required reviewers and Copilot code review approvals), and its triggers widen the attack surface; this repository does not use `pull_request_target` ([#2625](https://github.com/github/awesome-copilot/pull/2625)). It is deferred until maintainers have reviewed the approach with GitHub security. Until then, maintainers merge PRs once `submission-gate` passes. + +## Portability to MOS3 + +The #4184 automation is driven by repository-local configuration and self-contained scripts, so another repository such as MOS3 can adopt it by copying files and editing configuration rather than code. + +| Artifact | Phase | How to port | Repository-specific settings | +|---|---|---|---| +| `CODEOWNERS` with team owners | 1 | Copy the structure | Team names and path patterns | +| `.github/review-routing.yml` | 1 | Copy and edit | Reviewer pools, targets, escalation | +| `.github/risk-tiers.yml` | 2 | Copy and edit | Path and risk classification rules | +| `submission-gate` check | 2 | Copy the workflow and script | Required checks per tier; include `canvas-smoke-test` only where canvas extensions exist | +| `canvas-smoke-test` (`eng/canvas-smoke-test.mjs`, `canvas-smoke-test.yml`, `canvas-smoke-test-comment.yml`) | 3 | Copy as-is where the repository ships canvas extensions and `eng/materialize-plugins.mjs` | `CANVAS_PREVIEW_MIN_WIDTH` and `CANVAS_PREVIEW_MIN_HEIGHT` variables | +| `.github/review-metrics.yml`, `eng/review-metrics.mjs`, `.github/workflows/review-metrics.yml` | 3 | Copy as-is | State and risk labels, automation workflow list, tracking issue title | +| `eng/lib/review-automation-github.mjs` | 3 | Copy as-is | None (uses `GITHUB_TOKEN` or the `gh` CLI) | + +The phase 3 scripts depend only on Node.js 22 and `js-yaml`. Workflows gate automatic runs on `github.repository_owner == 'github'`; update that condition when porting. `.github/workflows/setup-labels.yml` creates the `review-metrics` label. + +### Other marketplaces: `microsoft/azure-dev-tools` + +[`microsoft/azure-dev-tools`](https://github.com/microsoft/azure-dev-tools) is another Copilot plugin marketplace (`copilot plugin marketplace add microsoft/azure-dev-tools`, marketplace ID `azure-dev-tools`). It ships canvas plugins such as `azure-functions-hosted-skills`, `azure-resources-query`, `azure-cost-health-check`, and `azure-sre-agent`, plus the skill-only `canvas-authoring` plugin. It can reuse the same review model: CODEOWNERS teams, `review-routing.yml`, risk tiers, `submission-gate`, `canvas-smoke-test`, and the metrics workflow. + +- **Maps directly:** + - Canvas smoke-test and preview evidence: module graph, capability summary, unsafe paths, binaries, and `assets/preview.png` checks. + - Risk tiers and `submission-gate` rules. + - Weekly metrics. +- **Differences:** + - The plugins are first-party and Microsoft-owned, so resource ownership comes from CODEOWNERS teams more than contributor front matter. Contributor-history signals matter less there. + - Releases are pinned to immutable per-plugin tags, while awesome-copilot materializes plugins from the default branch. Point the smoke test's install step at the plugin source for the release tag. Treat release-tag or version bumps as their own risk tier that always needs maintainer review. + - Plugins with no canvas extension, such as `canvas-authoring`, report `canvas-smoke-test` as skipped. The check's layout detection (`extensions//` plus a matching `plugins//plugin.json`) may need adjusting to that repository's structure. \ No newline at end of file diff --git a/eng/README.md b/eng/README.md index c0597aaf57..870a08b607 100644 --- a/eng/README.md +++ b/eng/README.md @@ -32,6 +32,14 @@ node eng/review-routing.mjs validate # validate .github/review-routing.yml node --test eng/review-routing.test.mjs # unit tests ``` +## Review automation + +See [docs/maintainers/canvas-evidence-and-metrics.md](../docs/maintainers/canvas-evidence-and-metrics.md) for details. + +- `canvas-smoke-test.mjs` โ€” static checks, preview validation, materialization, and install smoke test for canvas extensions (`canvas-smoke-test` check). +- `review-metrics.mjs` โ€” computes weekly review operating metrics and publishes them to the tracking issue. +- `lib/review-automation-github.mjs` โ€” small GitHub API client used by `review-metrics.mjs` (uses `GITHUB_TOKEN`, or the `gh` CLI locally). + ## Contributor Tools - `contributor-report.mjs` โ€” generates a markdown report of merged PRs for missing contributors (includes shared helpers). diff --git a/eng/canvas-smoke-test.mjs b/eng/canvas-smoke-test.mjs new file mode 100644 index 0000000000..cdd4bd6602 --- /dev/null +++ b/eng/canvas-smoke-test.mjs @@ -0,0 +1,1824 @@ +#!/usr/bin/env node + +// Canvas extension review evidence and smoke test. +// +// Static checks never execute extension code: modules are compiled (parsed) with +// node:vm but never linked or evaluated. The smoke test materializes affected +// plugins into a temporary copy of the repository with the real materializer +// and, when the Copilot CLI is available, installs them from an ephemeral local +// marketplace under an isolated COPILOT_HOME. + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import vm from "node:vm"; +import zlib from "node:zlib"; +import { builtinModules } from "node:module"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const DEFAULT_ROOT = path.join(path.dirname(__filename), ".."); + +export const REPORT_MARKER = ""; +// Lenient defaults so every preview already on main passes; raise them via +// CANVAS_PREVIEW_MIN_WIDTH / CANVAS_PREVIEW_MIN_HEIGHT once older previews are refreshed. +export const DEFAULT_MIN_PREVIEW_WIDTH = 400; +export const DEFAULT_MIN_PREVIEW_HEIGHT = 160; +export const MAX_PREVIEW_BYTES = 5 * 1024 * 1024; +export const MAX_ASSET_BYTES = 5 * 1024 * 1024; +const MAX_PNG_DIMENSION = 16384; +// Upper bound on decompressed image data so a small, highly compressed PNG +// cannot exhaust runner memory (8192ร—8192 RGBA8 is ~256 MiB). +export const MAX_PNG_DECODED_BYTES = 256 * 1024 * 1024; + +const COPILOT_NAMESPACE = "com.github.copilot"; +const AWESOME_COPILOT_NAMESPACE = "com.github.awesome-copilot"; +const HOST_PROVIDED_PACKAGES = ["@github/copilot-sdk"]; +const MODULE_EXTENSIONS = new Set([".mjs", ".js", ".cjs"]); +const IGNORED_DIRECTORIES = new Set([".git"]); + +const NATIVE_BINARY_EXTENSIONS = new Set([ + ".a", ".app", ".bin", ".class", ".com", ".deb", ".dll", ".dmg", ".dylib", ".exe", ".jar", + ".lib", ".msi", ".node", ".o", ".pyc", ".rpm", ".so", ".sys", ".wasm", +]); +const SCRIPT_EXTENSIONS = new Set([ + ".bash", ".bat", ".cmd", ".command", ".fish", ".pl", ".ps1", ".psm1", ".py", ".rb", ".sh", ".vbs", ".zsh", +]); +const ALLOWED_BINARY_EXTENSIONS = new Set([ + ".aac", ".apng", ".avif", ".bmp", ".flac", ".gif", ".ico", ".jpeg", ".jpg", ".m4a", ".mp3", ".mp4", + ".oga", ".ogg", ".opus", ".otf", ".png", ".ttf", ".wav", ".webm", ".webp", ".woff", ".woff2", +]); + +const BUILTIN_CAPABILITIES = [ + { modules: ["child_process"], label: "Spawns processes (node:child_process)" }, + { modules: ["fs", "fs/promises"], label: "File system access (node:fs)" }, + { modules: ["http", "https", "http2", "net", "tls", "dgram"], label: "Network sockets / local server (node:http, node:net, ...)" }, + { modules: ["worker_threads", "cluster"], label: "Worker threads / child processes (node:worker_threads, node:cluster)" }, + { modules: ["vm"], label: "Dynamic code evaluation (node:vm)" }, + { modules: ["os"], label: "Host information (node:os)" }, +]; +const SOURCE_CAPABILITIES = [ + { pattern: /\bfetch\s*\(/, label: "Outbound HTTP requests (fetch)" }, + { pattern: /\bprocess\.env\b/, label: "Reads environment variables (process.env)" }, + { pattern: /\beval\s*\(|\bnew\s+Function\s*\(/, label: "Dynamic code evaluation (eval / new Function)" }, + { pattern: /\bnew\s+WebSocket\s*\(/, label: "WebSocket connections" }, +]; + +const NODE_BUILTINS = new Set(builtinModules.map((name) => name.replace(/^node:/, ""))); + +function toPosix(value) { + return value.split(path.sep).join("/"); +} + +function isInside(parent, child) { + const relative = path.relative(parent, child); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function readJson(filePath, { root } = {}) { + try { + const read = readRegularFile(filePath, { root }); + if (read.error) return { error: `${path.basename(filePath)} ${read.error}` }; + return { value: JSON.parse(read.text) }; + } catch (error) { + return { error: error.message }; + } +} + +export const MAX_TEXT_FILE_BYTES = 5 * 1024 * 1024; + +/** + * Read a file without following symlinks, refusing devices, FIFOs, oversized + * files, and anything that resolves outside `root`. Returns { text } or { error }. + */ +export function readRegularFile(filePath, { root, maxBytes = MAX_TEXT_FILE_BYTES } = {}) { + let stat; + try { + stat = fs.lstatSync(filePath); + } catch (error) { + return { error: `cannot be read (${error.code ?? error.message})`, missing: error.code === "ENOENT" }; + } + if (stat.isSymbolicLink()) return { error: "is a symbolic link" }; + if (!stat.isFile()) return { error: "is not a regular file" }; + if (root && !isInside(fs.realpathSync(root), fs.realpathSync(filePath))) return { error: "resolves outside its directory" }; + if (stat.size > maxBytes) return { error: `is ${formatBytes(stat.size)}; the limit is ${formatBytes(maxBytes)}` }; + const fd = fs.openSync(filePath, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0)); + try { + const buffer = Buffer.alloc(stat.size); + let offset = 0; + while (offset < stat.size) { + const bytesRead = fs.readSync(fd, buffer, offset, stat.size - offset, offset); + if (bytesRead === 0) break; + offset += bytesRead; + } + return { text: buffer.subarray(0, offset).toString("utf8") }; + } finally { + fs.closeSync(fd); + } +} + +/** Why an extension directory cannot be inspected safely, or null. */ +function extensionDirProblem(extensionDir) { + const stat = fs.lstatSync(extensionDir, { throwIfNoEntry: false }); + if (!stat) return "extension directory is missing"; + if (stat.isSymbolicLink()) return "extension directory must not be a symbolic link"; + if (!stat.isDirectory()) return "extension path is not a directory"; + return null; +} + +// --------------------------------------------------------------------------- +// Path safety +// --------------------------------------------------------------------------- + +/** + * Classify a path reference found in a manifest or module specifier. + * Returns null when safe, otherwise a short reason. + */ +export function unsafePathReason(reference) { + if (typeof reference !== "string" || reference.length === 0) { + return null; + } + if (/^[A-Za-z]:[\\/]/.test(reference) || reference.startsWith("\\\\")) { + return "absolute Windows path"; + } + if (reference.startsWith("/")) { + return "absolute path"; + } + if (/^file:/i.test(reference)) { + return "file: URL"; + } + const segments = reference.split(/[\\/]+/); + if (segments.includes("..")) { + return "parent-directory (..) traversal"; + } + return null; +} + +function looksLikePath(value) { + if (typeof value !== "string" || value.length === 0 || value.length > 512) return false; + if (/^file:/i.test(value)) return true; + if (/\s/.test(value) || /^[a-z][a-z0-9+.-]*:\/\//i.test(value)) return false; + return /^(\.{1,2}[\\/]|\/|[A-Za-z]:[\\/]|\\\\)/.test(value) || /(^|[\\/])\.\.([\\/]|$)/.test(value); +} + +/** + * Walk a parsed manifest and report path-like string values that are absolute + * or traverse outside the containing directory. + */ +export function findUnsafeManifestPaths(value, trail = "$") { + const findings = []; + if (typeof value === "string") { + if (looksLikePath(value)) { + const reason = unsafePathReason(value); + if (reason) findings.push({ field: trail, value, reason }); + } + } else if (Array.isArray(value)) { + value.forEach((item, index) => findings.push(...findUnsafeManifestPaths(item, `${trail}[${index}]`))); + } else if (value && typeof value === "object") { + for (const [key, item] of Object.entries(value)) { + findings.push(...findUnsafeManifestPaths(item, `${trail}.${key}`)); + } + } + return findings; +} + +// --------------------------------------------------------------------------- +// PNG inspection +// --------------------------------------------------------------------------- + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const PNG_CHANNELS = { 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 }; +const PNG_VALID_DEPTHS = { 0: [1, 2, 4, 8, 16], 2: [8, 16], 3: [1, 2, 4, 8], 4: [8, 16], 6: [8, 16] }; +const ADAM7_PASSES = [ + [0, 0, 8, 8], [4, 0, 8, 8], [0, 4, 4, 8], [2, 0, 4, 4], [0, 2, 2, 4], [1, 0, 2, 2], [0, 1, 1, 2], +]; + +let crcTable = null; +function crc32(buffer) { + if (typeof zlib.crc32 === "function") return zlib.crc32(buffer) >>> 0; + if (!crcTable) { + crcTable = new Uint32Array(256); + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + crcTable[n] = c >>> 0; + } + } + let crc = 0xffffffff; + for (const byte of buffer) crc = crcTable[(crc ^ byte) & 0xff] ^ (crc >>> 8); + return (crc ^ 0xffffffff) >>> 0; +} + +function expectedPngDataLength(width, height, bitsPerPixel, interlace) { + const rowBytes = (w) => Math.ceil((w * bitsPerPixel) / 8); + if (interlace === 0) { + return height * (rowBytes(width) + 1); + } + let total = 0; + for (const [xStart, yStart, xStep, yStep] of ADAM7_PASSES) { + const passWidth = Math.ceil((width - xStart) / xStep); + const passHeight = Math.ceil((height - yStart) / yStep); + if (passWidth > 0 && passHeight > 0) total += passHeight * (rowBytes(passWidth) + 1); + } + return total; +} + +/** + * Structurally decode a PNG: signature, chunk CRCs, IHDR, PLTE/IDAT/IEND + * ordering, and full zlib inflation of image data with a size check. + */ +export function inspectPng(buffer) { + const result = { ok: false, errors: [], width: 0, height: 0, animated: false }; + if (!Buffer.isBuffer(buffer) || buffer.length < PNG_SIGNATURE.length + 12) { + result.errors.push("file is too small to be a PNG"); + return result; + } + if (!buffer.subarray(0, 8).equals(PNG_SIGNATURE)) { + result.errors.push("missing PNG signature (file is not a PNG)"); + return result; + } + + let offset = 8; + let header = null; + let sawPalette = false; + let sawEnd = false; + const idat = []; + let chunkIndex = 0; + + while (offset < buffer.length) { + if (offset + 12 > buffer.length) { + result.errors.push("truncated chunk header"); + return result; + } + const length = buffer.readUInt32BE(offset); + const type = buffer.toString("latin1", offset + 4, offset + 8); + const dataStart = offset + 8; + const dataEnd = dataStart + length; + if (dataEnd + 4 > buffer.length) { + result.errors.push(`truncated ${type} chunk`); + return result; + } + const expectedCrc = buffer.readUInt32BE(dataEnd); + if (crc32(buffer.subarray(offset + 4, dataEnd)) !== expectedCrc) { + result.errors.push(`CRC mismatch in ${type} chunk`); + return result; + } + const data = buffer.subarray(dataStart, dataEnd); + + if (chunkIndex === 0 && type !== "IHDR") { + result.errors.push("first chunk must be IHDR"); + return result; + } + if (type === "IHDR") { + if (header) { + result.errors.push("duplicate IHDR chunk"); + return result; + } + if (length !== 13) { + result.errors.push("IHDR chunk has invalid length"); + return result; + } + header = { + width: data.readUInt32BE(0), + height: data.readUInt32BE(4), + bitDepth: data[8], + colorType: data[9], + compression: data[10], + filter: data[11], + interlace: data[12], + }; + } else if (type === "PLTE") { + sawPalette = true; + } else if (type === "acTL") { + result.animated = true; + } else if (type === "IDAT") { + idat.push(data); + } else if (type === "IEND") { + if (length !== 0) { + result.errors.push("IEND chunk has invalid length"); + return result; + } + sawEnd = true; + offset = dataEnd + 4; + if (offset !== buffer.length) { + result.errors.push("data appears after the IEND chunk"); + return result; + } + break; + } + offset = dataEnd + 4; + chunkIndex++; + } + + if (!header) { + result.errors.push("missing IHDR chunk"); + return result; + } + Object.assign(result, { + width: header.width, + height: header.height, + bitDepth: header.bitDepth, + colorType: header.colorType, + interlaced: header.interlace === 1, + }); + + if (header.width === 0 || header.height === 0) result.errors.push("image has zero width or height"); + if (header.width > MAX_PNG_DIMENSION || header.height > MAX_PNG_DIMENSION) { + result.errors.push(`image dimensions exceed ${MAX_PNG_DIMENSION}px`); + } + if (!(header.colorType in PNG_CHANNELS) || !PNG_VALID_DEPTHS[header.colorType].includes(header.bitDepth)) { + result.errors.push(`invalid color type/bit depth (${header.colorType}/${header.bitDepth})`); + } + if (header.compression !== 0 || header.filter !== 0 || header.interlace > 1) { + result.errors.push("unsupported compression, filter, or interlace method"); + } + if (header.colorType === 3 && !sawPalette) result.errors.push("palette image is missing PLTE chunk"); + if (idat.length === 0) result.errors.push("missing IDAT image data"); + if (!sawEnd) result.errors.push("missing IEND chunk"); + if (result.errors.length > 0) return result; + + const bitsPerPixel = PNG_CHANNELS[header.colorType] * header.bitDepth; + const expected = expectedPngDataLength(header.width, header.height, bitsPerPixel, header.interlace); + if (expected > MAX_PNG_DECODED_BYTES) { + result.errors.push(`decoded image data would be ${formatBytes(expected)}; maximum is ${formatBytes(MAX_PNG_DECODED_BYTES)}`); + return result; + } + let inflated; + try { + inflated = zlib.inflateSync(Buffer.concat(idat), { maxOutputLength: expected }); + } catch (error) { + result.errors.push(error.code === "ERR_BUFFER_TOO_LARGE" + ? "image data is larger than the IHDR dimensions allow" + : `image data failed to decompress: ${error.message}`); + return result; + } + if (inflated.length < expected) { + result.errors.push(`image data is truncated (${inflated.length} of ${expected} bytes)`); + return result; + } + if (header.interlace === 0) { + const stride = Math.ceil((header.width * bitsPerPixel) / 8) + 1; + for (let row = 0; row < header.height; row++) { + if (inflated[row * stride] > 4) { + result.errors.push(`invalid scanline filter type on row ${row}`); + return result; + } + } + } + + result.ok = true; + return result; +} + +// --------------------------------------------------------------------------- +// Module parsing and import validation +// --------------------------------------------------------------------------- + +const CHILD_PARSER = ` +const vm = require("vm"); +const source = require("fs").readFileSync(0, "utf8"); +try { + const mod = new vm.SourceTextModule(source, { identifier: process.argv[1] || "module.mjs" }); + const specifiers = mod.moduleRequests ? mod.moduleRequests.map((r) => r.specifier) : mod.dependencySpecifiers; + process.stdout.write(JSON.stringify({ ok: true, specifiers: [...specifiers] })); +} catch (error) { + process.stdout.write(JSON.stringify({ ok: false, error: String(error && error.message || error) })); +} +`; + +function staticSpecifiersInProcess(source, identifier) { + const mod = new vm.SourceTextModule(source, { identifier }); + const specifiers = mod.moduleRequests ? mod.moduleRequests.map((request) => request.specifier) : mod.dependencySpecifiers; + return [...specifiers]; +} + +/** + * Compile an ES module without linking or evaluating it and return its static + * import specifiers. Falls back to a child process when vm modules are not + * enabled in the current process. + */ +export function parseEsModule(source, identifier = "module.mjs") { + if (typeof vm.SourceTextModule === "function") { + try { + return { ok: true, specifiers: staticSpecifiersInProcess(source, identifier) }; + } catch (error) { + return { ok: false, error: error.message }; + } + } + const child = spawnSync( + process.execPath, + ["--experimental-vm-modules", "--no-warnings", "-e", CHILD_PARSER, identifier], + { input: source, encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, + ); + try { + return JSON.parse(child.stdout); + } catch { + return { ok: false, error: `parser failed: ${(child.stderr || child.error?.message || "unknown error").trim()}` }; + } +} + +/** + * Compile a CommonJS script without running it and return its literal + * require() specifiers. + */ +export function parseCommonJs(source, identifier = "module.cjs") { + try { + vm.compileFunction(source.replace(/^#!.*/, ""), ["exports", "require", "module", "__filename", "__dirname"], { filename: identifier }); + return { ok: true, specifiers: findRequireSpecifiers(source) }; + } catch (error) { + return { ok: false, error: error.message }; + } +} + +const REGEX_PREFIX_CHARS = new Set("(,=:[!&|?{};+-*%<>~^".split("")); +const REGEX_PREFIX_WORDS = new Set(["return", "typeof", "instanceof", "case", "do", "else", "in", "of", "new", "delete", "void", "throw", "yield", "await"]); +const IDENTIFIER_CHAR = /[\w$]/; + +/** + * Blank out comments while leaving string, template, and regex literals + * intact, so `"a//b"` or `/\/\*x/` cannot hide the code that follows. + * With `maskLiterals`, literal contents are blanked too (delimiters kept), so + * call-shaped text inside data is not mistaken for code. + * Offsets and line breaks are preserved. + */ +export function stripComments(source, { maskLiterals = false } = {}) { + const n = source.length; + const blank = (text) => text.replace(/[^\n]/g, " "); + // Keep the first `open` and last `close` characters of a literal, masking the rest when asked. + const literal = (text, open, close) => + maskLiterals && text.length > open + close + ? text.slice(0, open) + blank(text.slice(open, text.length - close)) + text.slice(text.length - close) + : text; + const templateStack = []; + let out = ""; + let i = 0; + let braceDepth = 0; + let lastSignificant = ""; + let lastWord = ""; + + const scanTemplate = (start) => { + let j = start; + while (j < n) { + const c = source[j]; + if (c === "\\") { j += 2; continue; } + if (c === "`") { j++; out += literal(source.slice(start, j), 0, 1); lastSignificant = "`"; lastWord = ""; return j; } + if (c === "$" && source[j + 1] === "{") { + j += 2; + templateStack.push(braceDepth); + braceDepth++; + out += literal(source.slice(start, j), 0, 2); + lastSignificant = "{"; lastWord = ""; + return j; + } + j++; + } + out += literal(source.slice(start, n), 0, 0); + return n; + }; + + const regexAllowed = () => { + if (lastSignificant === "") return true; + if (IDENTIFIER_CHAR.test(lastSignificant)) return REGEX_PREFIX_WORDS.has(lastWord); + return REGEX_PREFIX_CHARS.has(lastSignificant); + }; + + while (i < n) { + const ch = source[i]; + const next = source[i + 1]; + if (ch === "/" && next === "/") { + let j = source.indexOf("\n", i); + if (j === -1) j = n; + out += blank(source.slice(i, j)); + i = j; + continue; + } + if (ch === "/" && next === "*") { + let j = source.indexOf("*/", i + 2); + j = j === -1 ? n : j + 2; + out += blank(source.slice(i, j)); + i = j; + continue; + } + if (ch === "'" || ch === '"') { + let j = i + 1; + while (j < n && source[j] !== ch && source[j] !== "\n") j += source[j] === "\\" ? 2 : 1; + j = Math.min(j + 1, n); + out += literal(source.slice(i, j), 1, source[j - 1] === ch && j - 1 > i ? 1 : 0); + i = j; + lastSignificant = ch; lastWord = ""; + continue; + } + if (ch === "`") { + out += ch; + i = scanTemplate(i + 1); + continue; + } + if (ch === "/" && regexAllowed()) { + let j = i + 1; + let inClass = false; + while (j < n && source[j] !== "\n") { + const c = source[j]; + if (c === "\\") { j += 2; continue; } + if (inClass) { if (c === "]") inClass = false; } + else if (c === "[") inClass = true; + else if (c === "/") { j++; break; } + j++; + } + while (j < n && /[a-z]/i.test(source[j])) j++; + out += literal(source.slice(i, j), 1, 0); + i = j; + lastSignificant = "a"; lastWord = ""; + continue; + } + if (ch === "{") braceDepth++; + if (ch === "}") { + braceDepth--; + if (templateStack.length && templateStack[templateStack.length - 1] === braceDepth) { + templateStack.pop(); + out += ch; + i = scanTemplate(i + 1); + continue; + } + } + if ((ch === "+" || ch === "-") && next === ch) { + // `x++ / y` is division, `++x / y` is not: a postfix update leaves an operand behind. + const postfix = !regexAllowed(); + out += ch + next; + i += 2; + lastSignificant = postfix ? ")" : ch; + lastWord = ""; + continue; + } + if (IDENTIFIER_CHAR.test(ch)) { + let j = i; + while (j < n && IDENTIFIER_CHAR.test(source[j])) j++; + lastWord = source.slice(i, j); + lastSignificant = source[j - 1]; + out += lastWord; + i = j; + continue; + } + if (!/\s/.test(ch)) { lastSignificant = ch; lastWord = ""; } + out += ch; + i++; + } + return out; +} + +function literalCallSpecifiers(source, pattern) { + const specifiers = new Set(); + let match; + // Match call shapes on masked text, then read each specifier from the original source at the same offsets. + const text = stripComments(source, { maskLiterals: true }); + while ((match = pattern.exec(text))) { + const open = match.index + match[0].indexOf(match[1]); + const close = text.indexOf(match[1], open + 1); + specifiers.add(source.slice(open + 1, close)); + } + return [...specifiers]; +} + +export function findDynamicImportSpecifiers(source) { + return literalCallSpecifiers(source, /\bimport\s*\(\s*(['"])([^'"\n]+)\1\s*\)/g); +} + +export function findRequireSpecifiers(source) { + return literalCallSpecifiers(source, /(? { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (IGNORED_DIRECTORIES.has(entry.name)) continue; + const fullPath = path.join(dir, entry.name); + files.push({ fullPath, entry }); + if (entry.isDirectory()) walk(fullPath); + } + }; + walk(rootDir); + return files; +} + +function gitFileModes(rootDir, relativeDir) { + const result = spawnSync("git", ["ls-files", "-s", "--", relativeDir], { cwd: rootDir, encoding: "utf8" }); + const modes = new Map(); + if (result.status !== 0) return modes; + for (const line of result.stdout.split("\n")) { + const match = line.match(/^(\d{6}) [0-9a-f]+ \d+\t(.+)$/); + if (match) modes.set(match[2], match[1]); + } + return modes; +} + +function hasNativeMagic(header) { + if (header.length < 4) return null; + if (header[0] === 0x7f && header[1] === 0x45 && header[2] === 0x4c && header[3] === 0x46) return "ELF executable"; + if (header[0] === 0x4d && header[1] === 0x5a) return "Windows PE executable"; + const magic = header.readUInt32BE(0); + if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe, 0xcafebabe].includes(magic)) return "Mach-O / fat binary"; + if (header[0] === 0x00 && header[1] === 0x61 && header[2] === 0x73 && header[3] === 0x6d) return "WebAssembly module"; + return null; +} + +function readHeader(filePath, size = 8192) { + const fd = fs.openSync(filePath, "r"); + try { + const buffer = Buffer.alloc(size); + const bytesRead = fs.readSync(fd, buffer, 0, size, 0); + return buffer.subarray(0, bytesRead); + } finally { + fs.closeSync(fd); + } +} + +/** + * Inspect an extension directory for symlinks, executables, native binaries, + * vendored dependencies, and oversized assets. + */ +export function inspectExtensionFiles(extensionDir, { rootDir = DEFAULT_ROOT, fileModes } = {}) { + const errors = []; + const warnings = []; + const inventory = []; + const relativeExtensionDir = toPosix(path.relative(rootDir, extensionDir)); + const dirProblem = extensionDirProblem(extensionDir); + if (dirProblem) { + errors.push(dirProblem); + return { errors, warnings, inventory }; + } + const modes = fileModes ?? gitFileModes(rootDir, relativeExtensionDir); + + for (const { fullPath, entry } of listFiles(extensionDir)) { + const relative = toPosix(path.relative(extensionDir, fullPath)); + if (entry.isSymbolicLink()) { + errors.push(`${relative}: symbolic links are not allowed in extensions`); + continue; + } + if (entry.isDirectory()) { + if (entry.name === "node_modules") errors.push(`${relative}/: vendored node_modules must not be committed`); + continue; + } + if (!entry.isFile()) continue; + if (relative.split("/").includes("node_modules")) continue; + + const stat = fs.statSync(fullPath); + const ext = path.extname(entry.name).toLowerCase(); + inventory.push({ path: relative, bytes: stat.size }); + + const repoRelative = toPosix(path.relative(rootDir, fullPath)); + const gitMode = modes.get(repoRelative); + const executableBit = gitMode ? gitMode === "100755" : process.platform !== "win32" && (stat.mode & 0o111) !== 0; + if (executableBit) errors.push(`${relative}: file is marked executable`); + + const header = readHeader(fullPath); + const nativeKind = hasNativeMagic(header); + if (nativeKind) { + errors.push(`${relative}: contains a ${nativeKind}`); + } else if (NATIVE_BINARY_EXTENSIONS.has(ext)) { + errors.push(`${relative}: native/compiled binary file type (${ext}) is not allowed`); + } else if (SCRIPT_EXTENSIONS.has(ext)) { + warnings.push(`${relative}: script file โ€” confirm it is not executed automatically`); + } else if (header.includes(0) && !ALLOWED_BINARY_EXTENSIONS.has(ext)) { + warnings.push(`${relative}: unexpected binary content (${ext || "no extension"})`); + } + if (!MODULE_EXTENSIONS.has(ext) && header.subarray(0, 2).toString("latin1") === "#!") { + warnings.push(`${relative}: has a shebang line โ€” confirm it is not executed automatically`); + } + if (stat.size > MAX_ASSET_BYTES) { + errors.push(`${relative}: ${formatBytes(stat.size)} exceeds the ${formatBytes(MAX_ASSET_BYTES)} asset limit`); + } + } + + return { errors, warnings, inventory }; +} + +function formatBytes(bytes) { + if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${bytes} B`; +} + +// --------------------------------------------------------------------------- +// Extension checks +// --------------------------------------------------------------------------- + +/** + * Parse every module in an extension, walk the import graph from + * extension.mjs, and validate each specifier. + */ +export function checkExtensionModules(extensionDir) { + const errors = []; + const warnings = []; + const builtins = new Set(); + const externalPackages = new Set(); + const sourceCapabilities = new Set(); + const packageJsonPath = path.join(extensionDir, "package.json"); + let packageJson = {}; + const dirProblem = extensionDirProblem(extensionDir); + if (dirProblem) { + errors.push(dirProblem); + return { errors, warnings, modules: [], builtins: [], externalPackages: [], sourceCapabilities: [], packageJson }; + } + if (fs.lstatSync(packageJsonPath, { throwIfNoEntry: false })) { + const parsed = readJson(packageJsonPath, { root: extensionDir }); + if (parsed.error) errors.push(`package.json: invalid (${parsed.error})`); + else packageJson = parsed.value ?? {}; + } + const packageType = packageJson.type === "module" ? "module" : "commonjs"; + + const entry = path.join(extensionDir, "extension.mjs"); + const entryStat = fs.lstatSync(entry, { throwIfNoEntry: false }); + if (!entryStat) { + errors.push("extension.mjs: entry point is missing"); + return { errors, warnings, modules: [], builtins: [], externalPackages: [], sourceCapabilities: [], packageJson }; + } + if (entryStat.isSymbolicLink() || !entryStat.isFile()) { + errors.push("extension.mjs: entry point must be a regular file (not a symbolic link or device)"); + return { errors, warnings, modules: [], builtins: [], externalPackages: [], sourceCapabilities: [], packageJson }; + } + + const moduleFiles = listFiles(extensionDir) + .filter(({ entry: dirent, fullPath }) => + dirent.isFile() && + MODULE_EXTENSIONS.has(path.extname(fullPath)) && + !toPosix(path.relative(extensionDir, fullPath)).split("/").includes("node_modules")) + .map(({ fullPath }) => fullPath); + + const parsedModules = new Map(); + const parseFile = (filePath) => { + if (parsedModules.has(filePath)) return parsedModules.get(filePath); + const relative = toPosix(path.relative(extensionDir, filePath)); + const read = readRegularFile(filePath, { root: extensionDir }); + if (read.error) { + const record = { path: relative, esm: true, source: "", ok: false, unreadable: true, error: `file ${read.error}`, specifiers: [], dynamic: [] }; + parsedModules.set(filePath, record); + return record; + } + const source = read.text; + const esm = isModuleFile(filePath, packageType); + const parsed = esm ? parseEsModule(source, relative) : parseCommonJs(source, relative); + const record = { + path: relative, + esm, + source, + ...parsed, + dynamic: findDynamicImportSpecifiers(source), + nonLiteralLoads: findNonLiteralRuntimeLoads(source), + }; + parsedModules.set(filePath, record); + return record; + }; + + const reachable = new Set(); + const queue = [entry]; + while (queue.length > 0) { + const filePath = queue.shift(); + if (reachable.has(filePath)) continue; + reachable.add(filePath); + const record = parseFile(filePath); + if (!record.ok) { + // Symlinked or non-regular targets are not in moduleFiles, so report them here. + if (record.unreadable && !moduleFiles.includes(filePath)) errors.push(`${record.path}: ${record.error}`); + continue; + } + for (const specifier of record.specifiers) { + for (const resolved of validateSpecifier(specifier, filePath, record.path, { strict: true, commonjs: !record.esm })) { + if (MODULE_EXTENSIONS.has(path.extname(resolved))) queue.push(resolved); + } + } + // Literal dynamic imports from reachable code are reachable too. + for (const specifier of record.dynamic) { + for (const resolved of validateSpecifier(specifier, filePath, record.path, { strict: true, dynamic: true })) { + if (MODULE_EXTENSIONS.has(path.extname(resolved))) queue.push(resolved); + } + } + } + + for (const filePath of moduleFiles) { + let record = parseFile(filePath); + const isReachable = reachable.has(filePath); + if (!record.ok && !isReachable && !record.unreadable) { + // Unreachable files are often browser assets served to the canvas webview, + // so accept them if they parse in either module flavour. + const alternate = record.esm ? parseCommonJs(record.source, record.path) : parseEsModule(record.source, record.path); + if (alternate.ok) record = { ...record, ...alternate, esm: !record.esm, error: undefined }; + } + if (!record.ok) { + errors.push(record.unreadable ? `${record.path}: ${record.error}` : `${record.path}: syntax error โ€” ${record.error}`); + continue; + } + for (const load of record.nonLiteralLoads) { + flag(isReachable, `${record.path}: non-literal ${load} cannot be analyzed safely`); + } + if (!isReachable) { + for (const specifier of record.specifiers) { + validateSpecifier(specifier, filePath, record.path, { strict: false, commonjs: !record.esm }); + } + for (const specifier of record.dynamic) { + validateDynamicSpecifier(specifier, filePath, record.path); + } + } + if (isReachable) { + for (const { pattern, label } of SOURCE_CAPABILITIES) { + if (pattern.test(record.source)) sourceCapabilities.add(label); + } + } + } + + // Problems in modules that are not reachable from extension.mjs are reported + // as warnings because those files may be browser assets rather than Node code. + function flag(strict, message) { + if (strict) errors.push(message); + else warnings.push(`${message} (module is not reachable from extension.mjs)`); + } + + // Returns the extension files the specifier resolves to (empty when it does not resolve to a local file). + function validateSpecifier(specifier, filePath, relativePath, { strict, dynamic = false, commonjs = false, viaAlias }) { + const classification = classifySpecifier(specifier, packageJson); + const shown = viaAlias ? `${viaAlias}" -> "${specifier}` : specifier; + const where = dynamic + ? `${relativePath}: dynamic import("${shown}")` + : commonjs ? `${relativePath}: require("${shown}")` : `${relativePath}: import "${shown}"`; + switch (classification.kind) { + case "relative": { + const cleaned = specifier.replace(/[?#].*$/, ""); + const target = path.resolve(path.dirname(filePath), cleaned); + if (!isInside(extensionDir, target)) { + flag(strict, `${where} escapes the extension directory`); + return []; + } + const resolved = commonjs ? resolveCommonJsTarget(target) : target; + if (!resolved || !fs.existsSync(resolved)) { + flag(strict, `${where} references a missing file`); + return []; + } + if (fs.statSync(resolved).isDirectory()) { + flag(strict, `${where} points at a directory (ES modules require a file path)`); + return []; + } + return [resolved]; + } + case "builtin": + if (strict) builtins.add(classification.name); + return []; + case "host": + return []; + case "internal": { + // Resolve package.json "imports" aliases through the same checks; fail closed on anything we cannot analyze. + const targets = viaAlias ? null : importsAliasTargets(packageJson.imports[specifier]); + if (!targets || targets.length === 0 || targets.some((target) => target.startsWith("#") || target.startsWith("../"))) { + flag(strict, `${where} uses a package.json "imports" alias that cannot be analyzed`); + return []; + } + const aliasBase = path.join(extensionDir, "package.json"); + return targets.flatMap((target) => + validateSpecifier(target, aliasBase, relativePath, { strict, dynamic, commonjs, viaAlias: specifier })); + } + case "dependency": + if (strict) externalPackages.add(classification.name); + return []; + case "dev-dependency": + if (strict) externalPackages.add(classification.name); + flag(strict, `${where} is only declared in devDependencies; runtime imports must be in dependencies`); + return []; + case "unsafe": + flag(strict, `${where} uses an unsafe ${classification.reason}`); + return []; + case "remote": + flag(strict, `${where} is a remote URL import`); + return []; + case "data": + flag(strict, `${where} is a data: URL import`); + return []; + default: + flag(strict, `${where} is not a Node.js builtin, host-provided package, or declared dependency`); + return []; + } + } + + // Dynamic imports in modules that are not reachable from extension.mjs + // (typically browser assets served to the canvas webview). + function validateDynamicSpecifier(specifier, filePath, relativePath) { + const where = `${relativePath}: dynamic import("${specifier}")`; + const classification = classifySpecifier(specifier, packageJson); + if (classification.kind === "relative") { + const target = path.resolve(path.dirname(filePath), specifier.replace(/[?#].*$/, "")); + if (!isInside(extensionDir, target)) flag(false, `${where} escapes the extension directory`); + else if (!fs.existsSync(target)) flag(false, `${where} references a missing file`); + } else if (classification.kind === "unsafe") { + flag(false, `${where} uses an unsafe ${classification.reason}`); + } else if (classification.kind === "remote") { + flag(false, `${where} loads remote code`); + } else if (classification.kind === "data") { + flag(false, `${where} is a data: URL import`); + } else if (classification.kind === "undeclared") { + flag(false, `${where} is not declared in package.json`); + } else if (classification.kind === "internal") { + validateSpecifier(specifier, filePath, relativePath, { strict: false, dynamic: true }); + } + } + + function resolveCommonJsTarget(target) { + const candidates = [target, `${target}.js`, `${target}.cjs`, `${target}.json`, path.join(target, "index.js"), path.join(target, "index.cjs")]; + return candidates.find((candidate) => fs.existsSync(candidate) && fs.statSync(candidate).isFile()) ?? null; + } + + if (packageJson.scripts) { + for (const hook of ["preinstall", "install", "postinstall", "prepare"]) { + if (packageJson.scripts[hook]) warnings.push(`package.json: defines a "${hook}" lifecycle script`); + } + } + if (typeof packageJson.main === "string") { + const reason = unsafePathReason(packageJson.main); + if (reason) errors.push(`package.json: "main" uses an unsafe ${reason}`); + else if (!fs.existsSync(path.join(extensionDir, packageJson.main))) errors.push(`package.json: "main" references a missing file (${packageJson.main})`); + } + + const modules = moduleFiles + .map((filePath) => ({ path: toPosix(path.relative(extensionDir, filePath)), reachable: reachable.has(filePath) })) + .sort((a, b) => a.path.localeCompare(b.path)); + + return { + errors, + warnings, + modules, + builtins: [...builtins].sort(), + externalPackages: [...externalPackages].sort(), + sourceCapabilities: [...sourceCapabilities].sort(), + packageJson, + }; +} + +export function describeCapabilities(builtins, sourceCapabilities, externalPackages) { + const capabilities = []; + for (const { modules, label } of BUILTIN_CAPABILITIES) { + if (modules.some((name) => builtins.includes(name))) capabilities.push(label); + } + capabilities.push(...sourceCapabilities); + if (externalPackages.length > 0) capabilities.push(`Third-party runtime packages: ${externalPackages.join(", ")}`); + return capabilities; +} + +export function checkPreview(extensionDir, { minWidth, minHeight }) { + const previewPath = path.join(extensionDir, "assets", "preview.png"); + const result = { path: "assets/preview.png", exists: false, errors: [], warnings: [] }; + const dirProblem = extensionDirProblem(extensionDir); + if (dirProblem) { + result.errors.push(dirProblem === "extension directory is missing" + ? "assets/preview.png is missing" + : "assets/preview.png cannot be inspected: extension directory is not a regular directory"); + return result; + } + const stat = fs.lstatSync(previewPath, { throwIfNoEntry: false }); + if (!stat) { + result.errors.push("assets/preview.png is missing"); + return result; + } + result.exists = true; + if (stat.isSymbolicLink() || !stat.isFile() || !isInside(fs.realpathSync(extensionDir), fs.realpathSync(previewPath))) { + result.errors.push("assets/preview.png must be a regular file"); + return result; + } + result.bytes = stat.size; + if (stat.size > MAX_PREVIEW_BYTES) { + result.errors.push(`assets/preview.png is ${formatBytes(stat.size)}; maximum is ${formatBytes(MAX_PREVIEW_BYTES)}`); + return result; + } + const png = inspectPng(fs.readFileSync(previewPath)); + result.width = png.width; + result.height = png.height; + result.animated = png.animated; + for (const error of png.errors) result.errors.push(`assets/preview.png: ${error}`); + if (png.ok && (png.width < minWidth || png.height < minHeight)) { + result.errors.push(`assets/preview.png is ${png.width}ร—${png.height}; minimum is ${minWidth}ร—${minHeight}`); + } + return result; +} + +// --------------------------------------------------------------------------- +// Target detection +// --------------------------------------------------------------------------- + +function readPluginManifests(rootDir) { + const pluginsDir = path.join(rootDir, "plugins"); + const manifests = new Map(); + if (!fs.existsSync(pluginsDir)) return manifests; + for (const entry of fs.readdirSync(pluginsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const manifestPath = path.join(pluginsDir, entry.name, "plugin.json"); + if (!fs.lstatSync(manifestPath, { throwIfNoEntry: false })) continue; + const parsed = readJson(manifestPath, { root: path.join(pluginsDir, entry.name) }); + manifests.set(entry.name, parsed.error ? { parseError: parsed.error } : parsed.value); + } + return manifests; +} + +function isExtensionDir(rootDir, name) { + return fs.existsSync(path.join(rootDir, "extensions", name, "extension.mjs")); +} + +const SAFE_EXTENSION_ID = /^[a-z0-9][a-z0-9._-]*$/i; + +/** A single path segment that stays inside extensions/ (no `..`, separators, or absolute paths). */ +export function isSafeExtensionId(id) { + return typeof id === "string" && SAFE_EXTENSION_ID.test(id) && !id.includes(".."); +} + +/** Extension references in a plugin manifest that are not safe single-segment IDs. */ +export function unsafeExtensionRefs(manifest) { + const refs = manifest?.extensions?.[AWESOME_COPILOT_NAMESPACE]?.extensions; + if (!Array.isArray(refs)) return []; + return refs.filter((ref) => typeof ref !== "string" + || !ref.startsWith("./extensions/") + || !isSafeExtensionId(ref.replace(/^\.\/extensions\//, "").replace(/\/$/, ""))); +} + +export function pluginExtensionIds(rootDir, pluginDir, manifest) { + const ids = new Set(); + const refs = manifest?.extensions?.[AWESOME_COPILOT_NAMESPACE]?.extensions; + if (Array.isArray(refs)) { + for (const ref of refs) { + if (typeof ref === "string" && ref.startsWith("./extensions/")) { + const id = ref.replace(/^\.\/extensions\//, "").replace(/\/$/, ""); + if (isSafeExtensionId(id)) ids.add(id); + } + } + } + if (isSafeExtensionId(pluginDir) && isExtensionDir(rootDir, pluginDir)) ids.add(pluginDir); + return [...ids].sort(); +} + +function readBasePluginManifest(rootDir, baseRef, pluginDir) { + if (!baseRef || !isSafeExtensionId(pluginDir)) return { error: `cannot read deleted plugins/${pluginDir}/plugin.json from base` }; + const result = spawnSync("git", ["show", `${baseRef}:plugins/${pluginDir}/plugin.json`], { + cwd: rootDir, + encoding: "utf8", + maxBuffer: MAX_TEXT_FILE_BYTES, + }); + if (result.status !== 0) { + const message = (result.stderr || result.stdout || `git show exited ${result.status}`).trim(); + return { error: `cannot read deleted plugins/${pluginDir}/plugin.json from base ${baseRef}: ${message}` }; + } + try { + return { value: JSON.parse(result.stdout) }; + } catch (error) { + return { error: `deleted plugins/${pluginDir}/plugin.json from base ${baseRef}: invalid JSON (${error.message})` }; + } +} + +/** + * Determine which canvas extensions and extension-bearing plugins are + * affected by a list of changed repository paths. + */ +export function detectCanvasTargets(changedFiles, { rootDir = DEFAULT_ROOT, baseRef = "" } = {}) { + const manifests = readPluginManifests(rootDir); + const extensions = new Set(); + const plugins = new Set(); + const removedExtensions = new Set(); + const baseManifestErrors = new Map(); + + for (const file of changedFiles) { + const parts = toPosix(file).split("/"); + if (parts[0] === "extensions" && parts.length >= 3) { + // A directory that still exists is validated even if extension.mjs was + // deleted, so removing the entry point cannot skip the check. + if (fs.existsSync(path.join(rootDir, "extensions", parts[1]))) extensions.add(parts[1]); + else removedExtensions.add(parts[1]); + } else if (parts[0] === "plugins" && parts.length >= 3) { + // A deleted plugin.json leaves no manifest, but the plugin is still + // checked when it has a direct extension so checkPluginManifest reports it. + let manifest = manifests.get(parts[1]); + const baseIds = new Set(); + if (parts[2] === "plugin.json" && baseRef) { + const baseManifest = readBasePluginManifest(rootDir, baseRef, parts[1]); + if (!manifest) { + if (baseManifest.error) { + baseManifestErrors.set(parts[1], baseManifest.error); + plugins.add(parts[1]); + } else { + manifest = baseManifest.value; + } + } else if (!baseManifest.error) { + // An edited manifest can drop a reference, which may leave the + // extension unregistered, so the base references are validated too. + // A missing or unparsable base copy just means there is nothing to + // compare against (for example a newly added manifest). + for (const id of pluginExtensionIds(rootDir, parts[1], baseManifest.value)) baseIds.add(id); + } + } + const ids = [...new Set([...(manifest ? pluginExtensionIds(rootDir, parts[1], manifest) : []), ...baseIds])]; + const directExtension = isSafeExtensionId(parts[1]) && isExtensionDir(rootDir, parts[1]); + const unsafeRefs = manifest ? unsafeExtensionRefs(manifest) : []; + if (ids.length > 0 || directExtension || unsafeRefs.length > 0) { + plugins.add(parts[1]); + ids.filter((id) => isExtensionDir(rootDir, id)).forEach((id) => extensions.add(id)); + if (directExtension) extensions.add(parts[1]); + } + } + } + + for (const [pluginDir, manifest] of manifests) { + const ids = pluginExtensionIds(rootDir, pluginDir, manifest); + // Plugins that still reference, or were the direct plugin for, a removed + // extension are validated so the removal cannot leave them broken. + if (ids.some((id) => extensions.has(id) || removedExtensions.has(id)) || removedExtensions.has(pluginDir)) { + plugins.add(pluginDir); + } + } + + return { + extensions: [...extensions].sort(), + plugins: [...plugins].sort(), + removedExtensions: [...removedExtensions].sort(), + manifests, + baseManifestErrors, + }; +} + +// --------------------------------------------------------------------------- +// Plugin checks, materialization, install smoke test +// --------------------------------------------------------------------------- + +function checkPluginManifest(rootDir, pluginDir, manifest) { + const errors = []; + const warnings = []; + if (!manifest) { + errors.push(`plugins/${pluginDir}/plugin.json is missing`); + return { errors, warnings }; + } + if (manifest.parseError) { + errors.push(`plugins/${pluginDir}/plugin.json: invalid JSON (${manifest.parseError})`); + return { errors, warnings }; + } + for (const finding of findUnsafeManifestPaths(manifest)) { + errors.push(`plugins/${pluginDir}/plugin.json ${finding.field}: unsafe ${finding.reason} (${finding.value})`); + } + for (const ref of unsafeExtensionRefs(manifest)) { + errors.push(`plugins/${pluginDir}/plugin.json extensions["${AWESOME_COPILOT_NAMESPACE}"].extensions: invalid extension reference ${JSON.stringify(ref)} (expected "./extensions/" with a single safe segment)`); + } + const logo = manifest.extensions?.[COPILOT_NAMESPACE]?.logo; + if (isExtensionDir(rootDir, pluginDir) && logo !== "assets/preview.png") { + errors.push(`plugins/${pluginDir}/plugin.json: extensions["${COPILOT_NAMESPACE}"].logo must be "assets/preview.png"`); + } + for (const id of pluginExtensionIds(rootDir, pluginDir, manifest)) { + if (!isExtensionDir(rootDir, id)) errors.push(`plugins/${pluginDir}/plugin.json references missing extension extensions/${id}`); + } + const composition = manifest.extensions?.[AWESOME_COPILOT_NAMESPACE] ?? {}; + for (const field of ["agents", "hooks", "skills"]) { + for (const ref of Array.isArray(composition[field]) ? composition[field] : []) { + const source = resolveCompositionSource(rootDir, ref); + if (!source || !fs.existsSync(source)) errors.push(`plugins/${pluginDir}/plugin.json ${field} reference not found: ${ref}`); + } + } + return { errors, warnings }; +} + +function resolveCompositionSource(rootDir, ref) { + if (typeof ref !== "string" || unsafePathReason(ref)) return null; + const trimmed = ref.replace(/^\.\//, "").replace(/\/$/, ""); + if (trimmed.startsWith("agents/")) return path.join(rootDir, "agents", `${path.basename(trimmed, ".md")}.agent.md`); + if (/^(skills|hooks|extensions)\//.test(trimmed)) return path.join(rootDir, trimmed); + return null; +} + +function copyPath(source, destination) { + fs.mkdirSync(path.dirname(destination), { recursive: true }); + fs.cpSync(source, destination, { recursive: true, dereference: false, verbatimSymlinks: true }); +} + +function commandAvailable(command) { + const probe = process.platform === "win32" + ? spawnSync("where", [command], { encoding: "utf8" }) + : spawnSync("sh", ["-c", `command -v ${command}`], { encoding: "utf8" }); + return probe.status === 0; +} + +function runCommand(command, args, options = {}) { + const result = spawnSync(command, args, { + encoding: "utf8", + timeout: 5 * 60 * 1000, + shell: process.platform === "win32", + ...options, + }); + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`.trim(); + return { status: typeof result.status === "number" ? result.status : 1, output: output.slice(-4000), error: result.error?.message }; +} + +function findFileUpwards(root, name, maxDepth = 4) { + const queue = [{ dir: root, depth: 0 }]; + while (queue.length > 0) { + const { dir, depth } = queue.shift(); + const candidate = path.join(dir, name); + if (fs.existsSync(candidate)) return candidate; + if (depth >= maxDepth) continue; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory() && entry.name !== "node_modules") queue.push({ dir: path.join(dir, entry.name), depth: depth + 1 }); + } + } + return null; +} + +/** + * Materialize the given plugins in an isolated copy of the repository with + * eng/materialize-plugins.mjs, verify the served output, and optionally + * install each plugin with the Copilot CLI from an ephemeral marketplace. + */ +export async function runSmokeTest(pluginDirs, { rootDir = DEFAULT_ROOT, workDir, install = "auto" } = {}) { + const results = { materialize: {}, install: {}, installStatus: "skipped", installNote: "" }; + if (pluginDirs.length === 0) return results; + + const tempRoot = workDir ?? fs.mkdtempSync(path.join(os.tmpdir(), "canvas-smoke-")); + const repoCopy = path.join(tempRoot, "repo"); + fs.rmSync(repoCopy, { recursive: true, force: true }); + copyPath(path.join(DEFAULT_ROOT, "eng", "materialize-plugins.mjs"), path.join(repoCopy, "eng", "materialize-plugins.mjs")); + copyPath(path.join(DEFAULT_ROOT, "eng", "constants.mjs"), path.join(repoCopy, "eng", "constants.mjs")); + + const { validateAgentPluginManifest } = await import("./agent-plugin-schema.mjs"); + const manifests = readPluginManifests(rootDir); + const copyable = []; + for (const pluginDir of pluginDirs) { + const manifest = manifests.get(pluginDir); + if (!manifest || manifest.parseError) { + results.materialize[pluginDir] = { status: "fail", errors: ["plugin.json is missing or invalid"] }; + continue; + } + if (!isSafeExtensionId(pluginDir) || unsafeExtensionRefs(manifest).length > 0) { + results.materialize[pluginDir] = { status: "fail", errors: ["plugin.json has unsafe plugin or extension references; not materialized"] }; + continue; + } + copyPath(path.join(rootDir, "plugins", pluginDir), path.join(repoCopy, "plugins", pluginDir)); + const composition = manifest.extensions?.[AWESOME_COPILOT_NAMESPACE] ?? {}; + for (const field of ["agents", "hooks", "skills"]) { + for (const ref of Array.isArray(composition[field]) ? composition[field] : []) { + const source = resolveCompositionSource(rootDir, ref); + if (source && fs.existsSync(source)) copyPath(source, path.join(repoCopy, path.relative(rootDir, source))); + } + } + for (const id of pluginExtensionIds(rootDir, pluginDir, manifest)) { + const source = path.join(rootDir, "extensions", id); + if (!isInside(path.join(rootDir, "extensions"), source)) continue; + if (fs.existsSync(source)) copyPath(source, path.join(repoCopy, "extensions", id)); + } + copyable.push(pluginDir); + } + + const materialize = runCommand(process.execPath, [path.join(repoCopy, "eng", "materialize-plugins.mjs")], { cwd: repoCopy, shell: false }); + for (const pluginDir of copyable) { + const errors = []; + const pluginRoot = path.join(repoCopy, "plugins", pluginDir); + if (materialize.status !== 0) errors.push(`materializer exited with ${materialize.status}: ${materialize.output}`); + const served = readJson(path.join(pluginRoot, "plugin.json")); + if (served.error) { + errors.push(`served plugin.json is invalid: ${served.error}`); + } else { + errors.push(...validateAgentPluginManifest(served.value).map((error) => `served plugin.json: ${error}`)); + if (served.value.extensions?.[AWESOME_COPILOT_NAMESPACE]) errors.push("served plugin.json still contains repository composition fields"); + } + const extensionIds = pluginExtensionIds(rootDir, pluginDir, manifests.get(pluginDir)); + for (const id of extensionIds) { + const materialized = path.join(pluginRoot, COPILOT_NAMESPACE, "extensions", id, "extension.mjs"); + if (!fs.existsSync(materialized)) errors.push(`materialized ${COPILOT_NAMESPACE}/extensions/${id}/extension.mjs is missing`); + } + results.materialize[pluginDir] = { status: errors.length ? "fail" : "pass", errors, extensions: extensionIds }; + } + + const canInstall = install !== "never" && commandAvailable("copilot"); + if (!canInstall) { + results.installStatus = install === "require" ? "infra_error" : "skipped"; + results.installNote = install === "never" ? "install smoke test disabled" : "Copilot CLI is not available on this runner"; + return results; + } + + const marketplaceName = "canvas-smoke-test"; + const marketplaceDir = path.join(tempRoot, "marketplace"); + fs.rmSync(marketplaceDir, { recursive: true, force: true }); + const entries = []; + for (const pluginDir of copyable) { + if (results.materialize[pluginDir].status !== "pass") continue; + copyPath(path.join(repoCopy, "plugins", pluginDir), path.join(marketplaceDir, "plugins", pluginDir)); + const served = readJson(path.join(repoCopy, "plugins", pluginDir, "plugin.json")).value; + entries.push({ name: served.name, source: `plugins/${pluginDir}`, description: served.description, version: served.version }); + } + const marketplace = { + name: marketplaceName, + metadata: { description: "Ephemeral marketplace for canvas smoke tests", version: "1.0.0" }, + owner: { name: "awesome-copilot", email: "noreply@github.com" }, + plugins: entries, + }; + fs.mkdirSync(path.join(marketplaceDir, ".github", "plugin"), { recursive: true }); + fs.writeFileSync(path.join(marketplaceDir, ".github", "plugin", "marketplace.json"), `${JSON.stringify(marketplace, null, 2)}\n`); + + const copilotHome = path.join(tempRoot, "copilot-home"); + fs.mkdirSync(copilotHome, { recursive: true }); + const env = { + ...process.env, + COPILOT_HOME: path.join(copilotHome, ".copilot"), + HOME: copilotHome, + XDG_CONFIG_HOME: path.join(copilotHome, ".config"), + XDG_CACHE_HOME: path.join(copilotHome, ".cache"), + XDG_DATA_HOME: path.join(copilotHome, ".local", "share"), + }; + delete env.GITHUB_TOKEN; + delete env.GH_TOKEN; + delete env.COPILOT_GITHUB_TOKEN; + + const add = runCommand("copilot", ["plugin", "marketplace", "add", marketplaceDir], { env }); + if (add.status !== 0) { + results.installStatus = "infra_error"; + results.installNote = `copilot plugin marketplace add failed: ${add.output || add.error}`; + return results; + } + + const outcomes = new Map(); + for (const entry of entries) { + outcomes.set(entry.name, runCommand("copilot", ["plugin", "install", `${entry.name}@${marketplaceName}`], { env })); + } + let listed = null; + const list = runCommand("copilot", ["plugin", "list", "--json"], { env }); + if (list.status === 0) { + try { + listed = JSON.parse(list.output.slice(list.output.indexOf("["))); + } catch { + listed = null; + } + } + + let failures = 0; + for (const entry of entries) { + const outcome = outcomes.get(entry.name); + const errors = []; + if (outcome.status !== 0) { + errors.push(`copilot plugin install failed: ${outcome.output || outcome.error}`); + } else { + // Older CLIs copy plugins into installed-plugins/; newer CLIs load local + // directory marketplaces live from their source directory. + const listing = Array.isArray(listed) + ? listed.find((item) => item.name === entry.name && item.marketplace === marketplaceName) + : null; + const copiedRoot = path.join(env.COPILOT_HOME, "installed-plugins", marketplaceName, entry.name); + const installedRoot = fs.existsSync(copiedRoot) + ? copiedRoot + : listing?.source === "live" ? path.join(marketplaceDir, entry.source) : null; + if (listed && !listing) errors.push("plugin is not listed by `copilot plugin list` after install"); + if (listing && listing.enabled === false) errors.push("plugin is installed but disabled"); + if (listing?.version && listing.version !== entry.version) errors.push(`installed version ${listing.version} does not match ${entry.version}`); + if (!installedRoot) { + errors.push(`installed plugin files not found (expected installed-plugins/${marketplaceName}/${entry.name} or a live listing)`); + } else { + if (!findFileUpwards(installedRoot, "plugin.json")) errors.push("installed plugin has no plugin.json"); + const pluginDir = entry.source.replace(/^plugins\//, ""); + for (const id of results.materialize[pluginDir].extensions) { + if (!fs.existsSync(path.join(installedRoot, COPILOT_NAMESPACE, "extensions", id, "extension.mjs"))) { + errors.push(`installed plugin is missing ${COPILOT_NAMESPACE}/extensions/${id}/extension.mjs`); + } + } + } + } + if (errors.length) failures++; + results.install[entry.source.replace(/^plugins\//, "")] = { status: errors.length ? "fail" : "pass", errors }; + } + results.installStatus = failures ? "fail" : "pass"; + return results; +} + +// --------------------------------------------------------------------------- +// Orchestration and reporting +// --------------------------------------------------------------------------- + +export async function runCanvasSmokeTest({ + rootDir = DEFAULT_ROOT, + changedFiles = [], + all = false, + minWidth = DEFAULT_MIN_PREVIEW_WIDTH, + minHeight = DEFAULT_MIN_PREVIEW_HEIGHT, + install = "auto", + workDir, + baseRef = "", +} = {}) { + const targets = all + ? (() => { + const extensionIds = fs.readdirSync(path.join(rootDir, "extensions"), { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && isExtensionDir(rootDir, entry.name)) + .map((entry) => entry.name); + return detectCanvasTargets(extensionIds.map((id) => `extensions/${id}/extension.mjs`), { rootDir, baseRef }); + })() + : detectCanvasTargets(changedFiles, { rootDir, baseRef }); + + const report = { + schema_version: "canvas-smoke-test/v1", + generated_at: new Date().toISOString(), + status: "skipped", + min_preview: { width: minWidth, height: minHeight }, + changed_files: changedFiles, + extensions: [], + plugins: [], + removed_extensions: targets.removedExtensions, + smoke: null, + error_count: 0, + warning_count: 0, + }; + if (targets.extensions.length === 0 && targets.plugins.length === 0) { + // A clean removal (extension and its plugin both deleted) is explicitly + // accepted rather than reported as skipped. + if (targets.removedExtensions.length > 0) report.status = "pass"; + return report; + } + + const owners = new Map(); + for (const [pluginDir, manifest] of targets.manifests) { + for (const id of pluginExtensionIds(rootDir, pluginDir, manifest)) { + owners.set(id, [...(owners.get(id) ?? []), pluginDir]); + } + } + + for (const id of targets.extensions) { + const extensionDir = path.join(rootDir, "extensions", id); + const modules = checkExtensionModules(extensionDir); + const files = inspectExtensionFiles(extensionDir, { rootDir }); + const preview = checkPreview(extensionDir, { minWidth, minHeight }); + const errors = [...new Set([...modules.errors, ...files.errors, ...preview.errors])]; + const warnings = [...modules.warnings, ...files.warnings, ...preview.warnings]; + + for (const manifestName of ["package.json", "copilot-extension.json"]) { + const manifestPath = path.join(extensionDir, manifestName); + if (!fs.lstatSync(manifestPath, { throwIfNoEntry: false })) continue; + const parsed = readJson(manifestPath, { root: extensionDir }); + if (parsed.error) { + if (manifestName !== "package.json") errors.push(`${manifestName}: invalid JSON (${parsed.error})`); + continue; + } + for (const finding of findUnsafeManifestPaths(parsed.value)) { + errors.push(`${manifestName} ${finding.field}: unsafe ${finding.reason} (${finding.value})`); + } + } + + const pluginDirs = (owners.get(id) ?? []).sort(); + if (pluginDirs.length === 0) errors.push(`extension is not registered by any plugin (add plugins/${id}/plugin.json or reference ./extensions/${id})`); + + report.extensions.push({ + id, + plugins: pluginDirs, + package: modules.packageJson?.name ? { name: modules.packageJson.name, version: modules.packageJson.version } : null, + modules: modules.modules, + builtins: modules.builtins, + third_party: modules.externalPackages, + capabilities: describeCapabilities(modules.builtins, modules.sourceCapabilities, modules.externalPackages), + preview, + files: files.inventory, + changed_files: changedFiles.filter((file) => toPosix(file).startsWith(`extensions/${id}/`)), + errors, + warnings, + }); + } + + for (const pluginDir of targets.plugins) { + const manifest = targets.manifests.get(pluginDir); + const check = checkPluginManifest(rootDir, pluginDir, manifest); + if (targets.baseManifestErrors.has(pluginDir)) { + check.errors.push(targets.baseManifestErrors.get(pluginDir)); + } + if (targets.removedExtensions.includes(pluginDir)) { + check.errors.push(`plugins/${pluginDir} is the plugin for removed extension extensions/${pluginDir}; delete the plugin too or restore the extension`); + } + report.plugins.push({ + directory: pluginDir, + name: manifest?.name ?? pluginDir, + version: manifest?.version ?? null, + description: manifest?.description ?? null, + author: manifest?.author ?? null, + license: manifest?.license ?? null, + keywords: Array.isArray(manifest?.keywords) ? manifest.keywords : [], + extensions: pluginExtensionIds(rootDir, pluginDir, manifest), + changed_files: changedFiles.filter((file) => toPosix(file).startsWith(`plugins/${pluginDir}/`)), + errors: check.errors, + warnings: check.warnings, + }); + } + + const ownsWorkDir = !workDir; + const smokeDir = workDir ?? fs.mkdtempSync(path.join(os.tmpdir(), "canvas-smoke-")); + try { + report.smoke = await runSmokeTest(targets.plugins, { rootDir, workDir: smokeDir, install }); + } finally { + if (ownsWorkDir) { + try { + fs.rmSync(smokeDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup of the temporary smoke-test directory. + } + } + } + + const smokeErrors = [ + ...Object.values(report.smoke.materialize).flatMap((entry) => entry.errors), + ...Object.values(report.smoke.install).flatMap((entry) => entry.errors), + ]; + report.error_count = smokeErrors.length + + report.extensions.reduce((sum, entry) => sum + entry.errors.length, 0) + + report.plugins.reduce((sum, entry) => sum + entry.errors.length, 0); + report.warning_count = report.extensions.reduce((sum, entry) => sum + entry.warnings.length, 0) + + report.plugins.reduce((sum, entry) => sum + entry.warnings.length, 0) + + (report.smoke.installStatus === "skipped" ? 1 : 0); + + if (report.error_count > 0) report.status = "fail"; + else if (report.smoke.installStatus === "infra_error") report.status = "infra_error"; + else report.status = "pass"; + return report; +} + +function escapeMarkdown(value) { + return String(value ?? "") + .replace(/[\r\n]+/g, " ") + .replace(/[<>]/g, (char) => (char === "<" ? "<" : ">")) + .replace(/([|`*_[\]])/g, "\\$1") + .slice(0, 300); +} + +function code(value) { + return `\`${String(value).replace(/`/g, "'").replace(/[\r\n]+/g, " ").slice(0, 300)}\``; +} + +function statusIcon(status) { + return { pass: "โœ…", fail: "โŒ", skipped: "โญ๏ธ", infra_error: "โš ๏ธ" }[status] ?? "โ”"; +} + +function formatAuthor(author) { + if (!author) return "โ€”"; + if (typeof author === "string") return escapeMarkdown(author); + return escapeMarkdown([author.name, author.url].filter(Boolean).join(" โ€” ")) || "โ€”"; +} + +function bulletList(items, empty = "_None_", limit = 50) { + if (!items || items.length === 0) return [empty]; + const lines = items.slice(0, limit).map((item) => `- ${item}`); + if (items.length > limit) lines.push(`- _โ€ฆand ${items.length - limit} more_`); + return lines; +} + +/** + * Render the Markdown review artifact used for the job summary and PR comment. + * previewBaseUrl, when provided, is prefixed to repository paths to embed the + * preview image (for example https://raw.githubusercontent.com////). + */ +export function renderMarkdownReport(report, { previewBaseUrl = "", runUrl = "" } = {}) { + const lines = [REPORT_MARKER, "## ๐Ÿงฉ Canvas smoke test", ""]; + if (report.status === "skipped") { + lines.push("โญ๏ธ **Skipped** โ€” no canvas extension or extension-bearing plugin paths changed."); + return `${lines.join("\n")}\n`; + } + if (report.extensions.length === 0 && report.plugins.length === 0 && report.removed_extensions.length > 0) { + lines.push(`โœ… **Passed** โ€” removed extensions ${report.removed_extensions.map(code).join(", ")}; no remaining plugin references them.`); + return `${lines.join("\n")}\n`; + } + const headline = { + pass: "โœ… **Passed**", + fail: "โŒ **Failed**", + infra_error: "โš ๏ธ **Infrastructure error** โ€” the smoke test could not complete; this is not a contribution failure.", + }[report.status]; + lines.push(`${headline} ยท ${report.error_count} error(s), ${report.warning_count} warning(s)`); + lines.push(""); + lines.push("| Check | Result |", "|---|---|"); + const allErrors = (items) => items.every((entry) => entry.errors.length === 0); + lines.push(`| Syntax, imports, file safety | ${allErrors(report.extensions) ? "โœ…" : "โŒ"} |`); + lines.push(`| Preview image (โ‰ฅ ${report.min_preview.width}ร—${report.min_preview.height} PNG) | ${report.extensions.every((entry) => entry.preview.errors.length === 0) ? "โœ…" : "โŒ"} |`); + lines.push(`| Plugin manifests | ${allErrors(report.plugins) ? "โœ…" : "โŒ"} |`); + const materializeStatuses = Object.values(report.smoke?.materialize ?? {}).map((entry) => entry.status); + lines.push(`| Materialize | ${materializeStatuses.length === 0 ? "โญ๏ธ" : materializeStatuses.every((status) => status === "pass") ? "โœ…" : "โŒ"} |`); + lines.push(`| Install (Copilot CLI) | ${statusIcon(report.smoke?.installStatus)}${report.smoke?.installNote ? ` ${escapeMarkdown(report.smoke.installNote)}` : ""} |`); + lines.push(""); + + for (const plugin of report.plugins) { + lines.push(`### Plugin ${code(plugin.name)}${plugin.version ? ` v${escapeMarkdown(plugin.version)}` : ""}`); + lines.push(""); + lines.push("| Field | Value |", "|---|---|"); + lines.push(`| Description | ${escapeMarkdown(plugin.description) || "โ€”"} |`); + lines.push(`| Author | ${formatAuthor(plugin.author)} |`); + lines.push(`| License | ${escapeMarkdown(plugin.license) || "โ€”"} |`); + lines.push(`| Keywords | ${plugin.keywords.map(escapeMarkdown).join(", ") || "โ€”"} |`); + lines.push(`| Extensions | ${plugin.extensions.map(code).join(", ") || "โ€”"} |`); + const materialize = report.smoke?.materialize?.[plugin.directory]; + const install = report.smoke?.install?.[plugin.directory]; + lines.push(`| Materialize | ${statusIcon(materialize?.status ?? "skipped")} |`); + lines.push(`| Install | ${statusIcon(install?.status ?? report.smoke?.installStatus ?? "skipped")} |`); + lines.push(""); + const problems = [...plugin.errors, ...(materialize?.errors ?? []), ...(install?.errors ?? [])]; + if (problems.length) lines.push("**Errors**", ...bulletList(problems.map(escapeMarkdown)), ""); + if (plugin.warnings.length) lines.push("**Warnings**", ...bulletList(plugin.warnings.map(escapeMarkdown)), ""); + if (plugin.changed_files.length) { + lines.push("
Changed plugin files", "", ...bulletList(plugin.changed_files.map(code)), "", "
", ""); + } + } + + for (const extension of report.extensions) { + lines.push(`### Extension ${code(extension.id)}`); + lines.push(""); + const preview = extension.preview; + if (preview.exists && previewBaseUrl && preview.errors.length === 0) { + lines.push(`${escapeMarkdown(extension.id)} preview`, ""); + } + lines.push("| Field | Value |", "|---|---|"); + lines.push(`| Registered by | ${extension.plugins.map(code).join(", ") || "โ€”"} |`); + lines.push(`| Package | ${extension.package ? `${code(extension.package.name)} ${escapeMarkdown(extension.package.version ?? "")}` : "โ€”"} |`); + lines.push(`| Preview | ${preview.exists ? `${preview.width}ร—${preview.height}${preview.animated ? " (animated)" : ""}, ${formatBytes(preview.bytes ?? 0)}` : "missing"} |`); + lines.push(`| Modules | ${extension.modules.length} (${extension.modules.filter((entry) => entry.reachable).length} reachable from ${code("extension.mjs")}) |`); + lines.push(`| Files | ${extension.files.length} |`); + lines.push(""); + lines.push("**Permissions and capabilities** (static analysis of modules reachable from `extension.mjs`)", ""); + lines.push(...bulletList(extension.capabilities.map(escapeMarkdown), "_No privileged Node.js capabilities detected_"), ""); + if (extension.errors.length) lines.push("**Errors**", ...bulletList(extension.errors.map(escapeMarkdown)), ""); + if (extension.warnings.length) lines.push("**Warnings**", ...bulletList(extension.warnings.map(escapeMarkdown)), ""); + if (extension.changed_files.length) { + lines.push("
Changed extension files", "", ...bulletList(extension.changed_files.map(code)), "", "
", ""); + } + } + + if (report.removed_extensions.length) { + lines.push(`Removed extensions: ${report.removed_extensions.map(code).join(", ")}`, ""); + } + lines.push("---"); + lines.push(`_Static checks compile modules without executing them. Minimum preview size is configurable with \`CANVAS_PREVIEW_MIN_WIDTH\`/\`CANVAS_PREVIEW_MIN_HEIGHT\`.${runUrl ? ` [Workflow run](${runUrl})` : ""}_`); + return `${lines.join("\n")}\n`; +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +function parseArgs(argv) { + const options = { + changedFiles: null, + all: false, + outputDir: null, + install: "auto", + minWidth: Number(process.env.CANVAS_PREVIEW_MIN_WIDTH) || DEFAULT_MIN_PREVIEW_WIDTH, + minHeight: Number(process.env.CANVAS_PREVIEW_MIN_HEIGHT) || DEFAULT_MIN_PREVIEW_HEIGHT, + previewBaseUrl: process.env.CANVAS_PREVIEW_BASE_URL || "", + runUrl: process.env.CANVAS_RUN_URL || "", + detectOnly: false, + baseRef: process.env.CANVAS_BASE_REF || "", + }; + for (let index = 0; index < argv.length; index++) { + const arg = argv[index]; + const next = () => argv[++index]; + if (arg === "--changed-files") options.changedFiles = next(); + else if (arg === "--all") options.all = true; + else if (arg === "--output-dir") options.outputDir = next(); + else if (arg === "--install") options.install = next(); + else if (arg === "--min-preview-width") options.minWidth = Number(next()); + else if (arg === "--min-preview-height") options.minHeight = Number(next()); + else if (arg === "--preview-base-url") options.previewBaseUrl = next(); + else if (arg === "--detect-only") options.detectOnly = true; + else if (arg === "--base-ref") options.baseRef = next(); + else if (arg === "--help" || arg === "-h") options.help = true; + else throw new Error(`Unknown argument: ${arg}`); + } + if (!["auto", "require", "never"].includes(options.install)) throw new Error("--install must be auto, require, or never"); + if (!(options.minWidth > 0) || !(options.minHeight > 0)) throw new Error("minimum preview dimensions must be positive numbers"); + return options; +} + +const USAGE = `Usage: node eng/canvas-smoke-test.mjs [--changed-files ] [--all] [--output-dir ] + [--install auto|require|never] [--min-preview-width ] [--min-preview-height ] + [--preview-base-url ] [--base-ref ] [--detect-only] + +Exit codes: 0 = passed or skipped, 1 = contribution failures, 2 = infrastructure error.`; + +async function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + console.log(USAGE); + return 0; + } + const changedFiles = options.changedFiles + ? fs.readFileSync(options.changedFiles, "utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean) + : []; + + if (options.detectOnly) { + const targets = detectCanvasTargets(changedFiles, { baseRef: options.baseRef }); + const canvas = targets.extensions.length > 0 || targets.plugins.length > 0 || targets.removedExtensions.length > 0; + console.log(JSON.stringify({ canvas, extensions: targets.extensions, plugins: targets.plugins, removedExtensions: targets.removedExtensions })); + if (process.env.GITHUB_OUTPUT) fs.appendFileSync(process.env.GITHUB_OUTPUT, `canvas=${canvas}\n`); + return 0; + } + + const report = await runCanvasSmokeTest({ + changedFiles, + all: options.all, + minWidth: options.minWidth, + minHeight: options.minHeight, + install: options.install, + baseRef: options.baseRef, + }); + const markdown = renderMarkdownReport(report, { previewBaseUrl: options.previewBaseUrl, runUrl: options.runUrl }); + + if (options.outputDir) { + fs.mkdirSync(path.join(options.outputDir, "previews"), { recursive: true }); + fs.writeFileSync(path.join(options.outputDir, "results.json"), `${JSON.stringify(report, null, 2)}\n`); + fs.writeFileSync(path.join(options.outputDir, "report.md"), markdown); + for (const extension of report.extensions) { + const source = path.join(DEFAULT_ROOT, "extensions", extension.id, "assets", "preview.png"); + if (extension.preview.exists && extension.preview.errors.length === 0) { + fs.copyFileSync(source, path.join(options.outputDir, "previews", `${extension.id}.png`)); + } + } + } + console.log(markdown); + if (report.status === "fail") return 1; + if (report.status === "infra_error") return 2; + return 0; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === __filename) { + main().then((code) => process.exit(code)).catch((error) => { + console.error(error.stack || error.message); + process.exit(2); + }); +} diff --git a/eng/canvas-smoke-test.test.mjs b/eng/canvas-smoke-test.test.mjs new file mode 100644 index 0000000000..fc3a189a9d --- /dev/null +++ b/eng/canvas-smoke-test.test.mjs @@ -0,0 +1,673 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import zlib from "node:zlib"; +import { test } from "node:test"; +import { + checkExtensionModules, + checkPreview, + classifySpecifier, + detectCanvasTargets, + findDynamicImportSpecifiers, + findNonLiteralRuntimeLoads, + findRequireSpecifiers, + findUnsafeManifestPaths, + importsAliasTargets, + inspectExtensionFiles, + inspectPng, + parseEsModule, + isSafeExtensionId, + readRegularFile, + renderMarkdownReport, + runCanvasSmokeTest, + stripComments, + unsafeExtensionRefs, + unsafePathReason, +} from "./canvas-smoke-test.mjs"; + +function crc32(buffer) { + let crc = 0xffffffff; + for (const byte of buffer) { + crc ^= byte; + for (let k = 0; k < 8; k++) crc = crc & 1 ? 0xedb88320 ^ (crc >>> 1) : crc >>> 1; + } + return (crc ^ 0xffffffff) >>> 0; +} + +function chunk(type, data) { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type, "latin1"), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(body)); + return Buffer.concat([length, body, crc]); +} + +function makePng(width, height, { truncate = false } = {}) { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; // bit depth + header[9] = 2; // truecolor + const rows = Buffer.alloc(height * (width * 3 + 1)); + const raw = truncate ? rows.subarray(0, rows.length / 2) : rows; + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk("IHDR", header), + chunk("IDAT", zlib.deflateSync(raw)), + chunk("IEND", Buffer.alloc(0)), + ]); +} + +function makeRepo(files) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "canvas-smoke-test-")); + for (const [relative, content] of Object.entries(files)) { + const target = path.join(root, relative); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, typeof content === "string" || Buffer.isBuffer(content) ? content : JSON.stringify(content, null, 2)); + } + return root; +} + +function git(root, args) { + const result = spawnSync("git", args, { cwd: root, encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr || result.stdout); + return result.stdout.trim(); +} + +const PLUGIN_SCHEMA = "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json"; + +function extensionPlugin(name) { + return { + $schema: PLUGIN_SCHEMA, + name, + description: `${name} canvas`, + version: "1.0.0", + extensions: { "com.github.copilot": { logo: "assets/preview.png" } }, + }; +} + +test("inspectPng accepts a valid PNG and reports dimensions", () => { + const result = inspectPng(makePng(32, 16)); + assert.equal(result.ok, true); + assert.equal(result.width, 32); + assert.equal(result.height, 16); +}); + +test("inspectPng rejects non-PNG, corrupted, and truncated data", () => { + assert.match(inspectPng(Buffer.from("GIF89a-not-a-png-at-all")).errors[0], /signature/); + + const corrupted = makePng(8, 8); + corrupted[corrupted.length - 20] ^= 0xff; + assert.equal(inspectPng(corrupted).ok, false); + + const truncated = inspectPng(makePng(8, 8, { truncate: true })); + assert.equal(truncated.ok, false); + assert.match(truncated.errors[0], /truncated/); +}); + +test("inspectPng rejects malformed IEND chunks and trailing data", () => { + const valid = makePng(8, 8); + assert.match(inspectPng(Buffer.concat([valid, Buffer.from("trailing")])).errors[0], /after the IEND/); + + const iendOffset = valid.length - 12; + const nonEmptyIend = Buffer.concat([ + valid.subarray(0, iendOffset), + chunk("IEND", Buffer.from([0])), + ]); + assert.match(inspectPng(nonEmptyIend).errors[0], /IEND chunk has invalid length/); +}); + +test("checkPreview enforces configurable minimum dimensions", () => { + const root = makeRepo({ "ext/assets/preview.png": makePng(100, 50) }); + const small = checkPreview(path.join(root, "ext"), { minWidth: 400, minHeight: 160 }); + assert.match(small.errors[0], /minimum is 400ร—160/); + const ok = checkPreview(path.join(root, "ext"), { minWidth: 100, minHeight: 50 }); + assert.deepEqual(ok.errors, []); + const missing = checkPreview(path.join(root, "missing"), { minWidth: 1, minHeight: 1 }); + assert.match(missing.errors[0], /missing/); +}); + +test("unsafePathReason flags absolute and traversal paths", () => { + assert.equal(unsafePathReason("./assets/preview.png"), null); + assert.equal(unsafePathReason("assets/preview.png"), null); + assert.match(unsafePathReason("../secrets.txt"), /traversal/); + assert.match(unsafePathReason("./a/../../b"), /traversal/); + assert.match(unsafePathReason("/etc/passwd"), /absolute/); + assert.match(unsafePathReason("C:\\Windows\\system32"), /Windows/); + assert.match(unsafePathReason("file:///tmp/x.mjs"), /file:/); +}); + +test("findUnsafeManifestPaths ignores prose and URLs", () => { + const findings = findUnsafeManifestPaths({ + description: "Reads ../ style docs and / separators in text", + homepage: "https://example.com/../x", + extensions: { "com.github.copilot": { logo: "../outside.png" } }, + main: "/abs/entry.mjs", + }); + assert.deepEqual(findings.map((finding) => finding.field).sort(), ["$.extensions.com.github.copilot.logo", "$.main"]); +}); + +test("classifySpecifier distinguishes builtins, host packages, dependencies, and unsafe specifiers", () => { + const pkg = { dependencies: { playwright: "1.0.0" }, devDependencies: { vitest: "1.0.0" } }; + assert.equal(classifySpecifier("node:fs", pkg).kind, "builtin"); + assert.equal(classifySpecifier("path", pkg).kind, "builtin"); + assert.equal(classifySpecifier("@github/copilot-sdk/extension", pkg).kind, "host"); + assert.equal(classifySpecifier("playwright/test", pkg).kind, "dependency"); + assert.equal(classifySpecifier("vitest", pkg).kind, "dev-dependency"); + assert.equal(classifySpecifier("left-pad", pkg).kind, "undeclared"); + assert.equal(classifySpecifier("/tmp/evil.mjs", pkg).kind, "unsafe"); + assert.equal(classifySpecifier("https://example.com/x.mjs", pkg).kind, "remote"); + assert.equal(classifySpecifier("./local.mjs", pkg).kind, "relative"); +}); + +test("parseEsModule returns static specifiers without executing code", () => { + const marker = path.join(os.tmpdir(), `canvas-smoke-exec-${process.pid}-${Date.now()}`); + const source = `import fs from "node:fs";\nimport { x } from "./x.mjs";\nfs.writeFileSync(${JSON.stringify(marker)}, "ran");\n`; + const result = parseEsModule(source, "entry.mjs"); + assert.equal(result.ok, true); + assert.deepEqual(result.specifiers.sort(), ["./x.mjs", "node:fs"]); + assert.equal(fs.existsSync(marker), false); + + const broken = parseEsModule("export const = 1;", "broken.mjs"); + assert.equal(broken.ok, false); +}); + +test("checkExtensionModules reports syntax errors, missing files, traversal, and undeclared packages", () => { + const root = makeRepo({ + "extensions/bad/extension.mjs": [ + 'import { joinSession } from "@github/copilot-sdk/extension";', + 'import { spawn } from "node:child_process";', + 'import helper from "./lib/helper.mjs";', + 'import missing from "./lib/missing.mjs";', + 'import outside from "../other/secret.mjs";', + 'import pad from "left-pad";', + "export default { joinSession, spawn, helper, missing, outside, pad };", + ].join("\n"), + "extensions/bad/lib/helper.mjs": "export default fetch;\nexport const broken = ;\n", + "extensions/bad/public/app.js": 'import { h } from "/vendor/preact.js";\nexport default h;\n', + "extensions/bad/package.json": { name: "bad", version: "1.0.0" }, + "extensions/other/secret.mjs": "export default 1;\n", + }); + const result = checkExtensionModules(path.join(root, "extensions", "bad")); + const text = result.errors.join("\n"); + assert.match(text, /lib\/helper\.mjs: syntax error/); + assert.match(text, /\.\/lib\/missing\.mjs" references a missing file/); + assert.match(text, /escapes the extension directory/); + assert.match(text, /"left-pad" is not a Node\.js builtin/); + assert.doesNotMatch(text, /public\/app\.js/); + assert.match(result.warnings.join("\n"), /public\/app\.js: import "\/vendor\/preact\.js" uses an unsafe absolute path \(module is not reachable/); + assert.deepEqual(result.builtins, ["child_process"]); +}); + +test("inspectExtensionFiles flags native binaries, executables, and vendored node_modules", () => { + const elf = Buffer.concat([Buffer.from([0x7f, 0x45, 0x4c, 0x46]), Buffer.alloc(32)]); + const root = makeRepo({ + "extensions/bin/extension.mjs": "export {};\n", + "extensions/bin/tool": elf, + "extensions/bin/addon.node": "not really native", + "extensions/bin/run.sh": "#!/bin/sh\necho hi\n", + "extensions/bin/assets/preview.png": makePng(4, 4), + "extensions/bin/node_modules/dep/index.js": "module.exports = 1;\n", + "extensions/bin/data.bin.dat": Buffer.from([0, 1, 2, 3]), + "extensions/bin/tool.py": "print(1)\n", + }); + const modes = new Map([["extensions/bin/run.sh", "100755"]]); + const result = inspectExtensionFiles(path.join(root, "extensions", "bin"), { rootDir: root, fileModes: modes }); + const errors = result.errors.join("\n"); + assert.match(errors, /tool: contains a ELF executable/); + assert.match(errors, /addon\.node: native\/compiled binary/); + assert.match(errors, /run\.sh: file is marked executable/); + assert.match(errors, /node_modules\/: vendored node_modules/); + assert.match(result.warnings.join("\n"), /run\.sh: script file/); + assert.match(result.warnings.join("\n"), /data\.bin\.dat: unexpected binary content/); + assert.match(result.warnings.join("\n"), /tool\.py: script file/); + assert.doesNotMatch(result.warnings.join("\n"), /preview\.png/); +}); + +test("detectCanvasTargets maps changed paths to extensions and extension-bearing plugins", () => { + const root = makeRepo({ + "extensions/orb/extension.mjs": "export {};\n", + "extensions/shared/extension.mjs": "export {};\n", + "plugins/orb/plugin.json": extensionPlugin("orb"), + "plugins/bundle/plugin.json": { + $schema: PLUGIN_SCHEMA, + name: "bundle", + description: "bundle", + version: "1.0.0", + extensions: { "com.github.awesome-copilot": { extensions: ["./extensions/shared"] } }, + }, + "plugins/plain/plugin.json": { $schema: PLUGIN_SCHEMA, name: "plain", description: "plain", version: "1.0.0" }, + }); + + assert.deepEqual(detectCanvasTargets(["README.md", "plugins/plain/README.md", "skills/x/SKILL.md"], { rootDir: root }).extensions, []); + + const orb = detectCanvasTargets(["extensions/orb/extension.mjs"], { rootDir: root }); + assert.deepEqual(orb.extensions, ["orb"]); + assert.deepEqual(orb.plugins, ["orb"]); + + const bundle = detectCanvasTargets(["plugins/bundle/plugin.json", "extensions/gone/extension.mjs"], { rootDir: root }); + assert.deepEqual(bundle.extensions, ["shared"]); + assert.deepEqual(bundle.plugins, ["bundle"]); + assert.deepEqual(bundle.removedExtensions, ["gone"]); +}); + +test("runCanvasSmokeTest reports skipped when no canvas paths change", async () => { + const root = makeRepo({ "plugins/plain/plugin.json": { name: "plain" } }); + const report = await runCanvasSmokeTest({ rootDir: root, changedFiles: ["docs/README.md"], install: "never" }); + assert.equal(report.status, "skipped"); + assert.match(renderMarkdownReport(report), /Skipped/); +}); + +test("runCanvasSmokeTest materializes a valid extension plugin and renders evidence", async () => { + const root = makeRepo({ + "extensions/orb/extension.mjs": 'import { joinSession } from "@github/copilot-sdk/extension";\nimport http from "node:http";\nexport default { joinSession, http };\n', + "extensions/orb/package.json": { name: "orb", version: "1.0.0", type: "module" }, + "extensions/orb/assets/preview.png": makePng(800, 400), + "plugins/orb/plugin.json": extensionPlugin("orb"), + "plugins/orb/README.md": "# Orb\n", + }); + const report = await runCanvasSmokeTest({ rootDir: root, changedFiles: ["extensions/orb/extension.mjs"], install: "never" }); + assert.equal(report.status, "pass", JSON.stringify(report, null, 2)); + assert.equal(report.smoke.materialize.orb.status, "pass"); + assert.equal(report.smoke.installStatus, "skipped"); + assert.equal(report.extensions[0].preview.width, 800); + + const markdown = renderMarkdownReport(report, { previewBaseUrl: "https://raw.githubusercontent.com/o/r/sha/" }); + assert.match(markdown, //); + assert.match(markdown, /Network sockets/); + assert.match(markdown, /extensions\/orb\/assets\/preview\.png/); +}); + +test("runCanvasSmokeTest fails for an unregistered extension with a missing preview", async () => { + const root = makeRepo({ "extensions/lonely/extension.mjs": "export {};\n" }); + const report = await runCanvasSmokeTest({ rootDir: root, changedFiles: ["extensions/lonely/extension.mjs"], install: "never" }); + assert.equal(report.status, "fail"); + const errors = report.extensions[0].errors.join("\n"); + assert.match(errors, /preview\.png is missing/); + assert.match(errors, /not registered by any plugin/); +}); + +test("inspectPng rejects images whose decoded data exceeds the budget before inflating", () => { + const header = Buffer.alloc(13); + header.writeUInt32BE(16000, 0); + header.writeUInt32BE(16000, 4); + header[8] = 16; // bit depth + header[9] = 6; // RGBA + const png = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk("IHDR", header), + chunk("IDAT", zlib.deflateSync(Buffer.alloc(1024))), + chunk("IEND", Buffer.alloc(0)), + ]); + const result = inspectPng(png); + assert.equal(result.ok, false); + assert.match(result.errors[0], /decoded image data would be/); +}); + +test("inspectPng rejects image data larger than IHDR allows", () => { + const header = Buffer.alloc(13); + header.writeUInt32BE(4, 0); + header.writeUInt32BE(4, 4); + header[8] = 8; + header[9] = 2; + const png = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk("IHDR", header), + chunk("IDAT", zlib.deflateSync(Buffer.alloc(10_000))), + chunk("IEND", Buffer.alloc(0)), + ]); + assert.match(inspectPng(png).errors[0], /larger than the IHDR/); +}); + +test("findUnsafeManifestPaths reports file: URLs", () => { + const findings = findUnsafeManifestPaths({ main: "file:///tmp/x.mjs", homepage: "https://example.com/x" }); + assert.deepEqual(findings.map((finding) => finding.field), ["$.main"]); + assert.match(findings[0].reason, /file:/); +}); + +test("checkExtensionModules follows CommonJS requires and literal dynamic imports", () => { + const root = makeRepo({ + "extensions/graph/extension.mjs": [ + 'import cjs from "./lib/legacy.cjs";', + 'const worker = await import("./lib/worker.mjs");', + 'const later = await import("./lib/missing.mjs");', + "// import(\"./lib/commented.mjs\")", + "export default { cjs, worker, later };", + ].join("\n"), + "extensions/graph/lib/legacy.cjs": [ + 'const helper = require("./helper");', + 'const pad = require("left-pad");', + 'const outside = require("../../other/x.js");', + "// require(\"ignored-in-comment\")", + "module.exports = { helper, pad, outside };", + ].join("\n"), + "extensions/graph/lib/helper.js": "module.exports = 1;\n", + "extensions/graph/lib/worker.mjs": 'import { spawn } from "node:child_process";\nimport vitest from "vitest";\nexport default { spawn, vitest };\n', + "extensions/graph/package.json": { name: "graph", version: "1.0.0", devDependencies: { vitest: "1.0.0" } }, + "extensions/other/x.js": "module.exports = 1;\n", + }); + + const result = checkExtensionModules(path.join(root, "extensions", "graph")); + const errors = result.errors.join("\n"); + assert.match(errors, /legacy\.cjs: require\("left-pad"\) is not a Node\.js builtin/); + assert.match(errors, /legacy\.cjs: require\("\.\.\/\.\.\/other\/x\.js"\) escapes the extension directory/); + assert.doesNotMatch(errors, /require\("\.\/helper"\)/); + assert.doesNotMatch(errors, /ignored-in-comment|commented\.mjs/); + assert.match(errors, /dynamic import\("\.\/lib\/missing\.mjs"\) references a missing file/); + assert.match(errors, /worker\.mjs: import "vitest" is only declared in devDependencies/); + assert.deepEqual(result.builtins, ["child_process"]); + assert.ok(result.modules.find((entry) => entry.path === "lib/worker.mjs").reachable); + assert.ok(result.modules.find((entry) => entry.path === "lib/helper.js").reachable); +}); + +test("checkExtensionModules fails reachable data URL imports and warns for unreachable dynamic data imports", () => { + const root = makeRepo({ + "extensions/data/extension.mjs": [ + 'import inline from "data:text/javascript,export default 1";', + 'const later = await import("data:text/javascript,export default 2");', + "export default { inline, later };", + ].join("\n"), + "extensions/data/public/app.js": 'await import("data:text/javascript,export default 3");\n', + }); + const result = checkExtensionModules(path.join(root, "extensions", "data")); + const errors = result.errors.join("\n"); + assert.match(errors, /extension\.mjs: import "data:text\/javascript,export default 1" is a data: URL import/); + assert.match(errors, /extension\.mjs: dynamic import\("data:text\/javascript,export default 2"\) is a data: URL import/); + assert.match(result.warnings.join("\n"), /public\/app\.js: dynamic import\("data:text\/javascript,export default 3"\) is a data: URL import \(module is not reachable/); +}); + +test("removed canvas paths are validated instead of skipped", async () => { + // Entry point deleted but the extension directory remains. + const partial = makeRepo({ + "extensions/orb/assets/preview.png": makePng(800, 400), + "plugins/orb/plugin.json": extensionPlugin("orb"), + }); + const partialReport = await runCanvasSmokeTest({ rootDir: partial, changedFiles: ["extensions/orb/extension.mjs"], install: "never" }); + assert.equal(partialReport.status, "fail"); + assert.match(partialReport.extensions[0].errors.join("\n"), /extension\.mjs: entry point is missing/); + + // Extension deleted, but its direct plugin and a bundling plugin remain. + const orphaned = makeRepo({ + "plugins/orb/plugin.json": extensionPlugin("orb"), + "plugins/bundle/plugin.json": { + $schema: PLUGIN_SCHEMA, + name: "bundle", + description: "bundle", + version: "1.0.0", + extensions: { "com.github.awesome-copilot": { extensions: ["./extensions/orb"] } }, + }, + }); + const orphanedTargets = detectCanvasTargets(["extensions/orb/extension.mjs"], { rootDir: orphaned }); + assert.deepEqual(orphanedTargets.plugins, ["bundle", "orb"]); + const orphanedReport = await runCanvasSmokeTest({ rootDir: orphaned, changedFiles: ["extensions/orb/extension.mjs"], install: "never" }); + assert.equal(orphanedReport.status, "fail"); + const pluginErrors = orphanedReport.plugins.flatMap((plugin) => plugin.errors).join("\n"); + assert.match(pluginErrors, /plugins\/orb is the plugin for removed extension/); + assert.match(pluginErrors, /plugins\/bundle\/plugin\.json references missing extension extensions\/orb/); + + // Extension and plugin both deleted: accepted. + const clean = makeRepo({ "plugins/plain/plugin.json": { name: "plain" } }); + const cleanReport = await runCanvasSmokeTest({ + rootDir: clean, + changedFiles: ["extensions/orb/extension.mjs", "plugins/orb/plugin.json"], + install: "never", + }); + assert.equal(cleanReport.status, "pass"); + assert.match(renderMarkdownReport(cleanReport), /removed extensions/); +}); + +test("stripComments keeps string, template, and regex literals that contain comment markers", () => { + const source = [ + 'const s = "x//y"; await import("./a.mjs");', + "const t = `/* ${'//'} */`; await import(\"./b.mjs\");", + 'const r = /\\/\\*/; await import("./c.mjs");', + '// await import("./hidden.mjs");', + '/* await import("./hidden2.mjs"); */', + 'const q = a / b; const w = c / d; await import("./d.mjs");', + ].join("\n"); + const text = stripComments(source); + for (const name of ["a", "b", "c", "d"]) assert.match(text, new RegExp(`import\\("\\./${name}\\.mjs"\\)`)); + assert.doesNotMatch(text, /hidden/); + assert.equal(text.split("\n").length, source.split("\n").length); +}); + +test("checkExtensionModules follows imports placed after strings containing //", () => { + const root = makeRepo({ + "extensions/str/extension.mjs": 'const s = "x//y"; const w = await import("./worker.mjs");\nexport default { s, w };\n', + "extensions/str/worker.mjs": 'import pad from "left-pad";\nexport default pad;\n', + }); + const result = checkExtensionModules(path.join(root, "extensions", "str")); + assert.match(result.errors.join("\n"), /worker\.mjs: import "left-pad" is not a Node\.js builtin/); + assert.ok(result.modules.find((entry) => entry.path === "worker.mjs").reachable); +}); + +test("inspectPng rejects a duplicate IHDR chunk", () => { + const valid = makePng(4, 4); + const ihdr = valid.subarray(8, 8 + 25); + const png = Buffer.concat([valid.subarray(0, 33), ihdr, valid.subarray(33)]); + const result = inspectPng(png); + assert.equal(result.ok, false); + assert.match(result.errors.join("\n"), /duplicate IHDR chunk/); +}); + +test("dynamic import and require extraction ignores call-shaped text inside literals", () => { + const source = [ + 'const a = \'import("./in-single.mjs")\';', + 'const b = "require(\\"./in-double.cjs\\")";', + "const c = `import(\"./in-template.mjs\") ${await import(\"./in-expr.mjs\")}`;", + 'const d = /import\\("\\.\\/in-regex\\.mjs"\\)/;', + 'const e = await import("./real.mjs"); const f = require("./real.cjs");', + ].join("\n"); + assert.deepEqual(findDynamicImportSpecifiers(source).sort(), ["./in-expr.mjs", "./real.mjs"]); + assert.deepEqual(findRequireSpecifiers(source), ["./real.cjs"]); + const root = makeRepo({ + "extensions/lit/extension.mjs": 'const note = \'import("./missing.mjs")\';\nexport default { note };\n', + }); + assert.deepEqual(checkExtensionModules(path.join(root, "extensions", "lit")).errors, []); +}); + +test("non-literal runtime loads are detected outside comments and literals", () => { + const source = [ + 'const literalImport = import("./literal.mjs");', + 'const literalRequire = require("left-pad");', + 'const computedImport = import("node:" + moduleName);', + "const computedRequire = require(moduleName);", + 'const text = "require(hidden)";', + "// import(variable)", + "const templateImport = import(`./${name}.mjs`);", + ].join("\n"); + assert.deepEqual(findNonLiteralRuntimeLoads(source), ["dynamic import", "dynamic import", "require"]); + + const root = makeRepo({ + "extensions/runtime/extension.mjs": 'const name = "fs";\nexport default import("node:" + name);\n', + }); + assert.match( + checkExtensionModules(path.join(root, "extensions", "runtime")).errors.join("\n"), + /extension\.mjs: non-literal dynamic import cannot be analyzed safely/, + ); +}); + +test("checkExtensionModules resolves package.json imports aliases and fails closed on unknown shapes", () => { + assert.deepEqual(importsAliasTargets({ node: "./a.mjs", default: { import: "./b.mjs" } }), ["./a.mjs", "./b.mjs"]); + assert.equal(importsAliasTargets(["./a.mjs"]), null); + const root = makeRepo({ + "extensions/alias/extension.mjs": [ + 'import ok from "#ok";', + 'import gone from "#gone";', + 'import pad from "#pad";', + 'import odd from "#odd";', + 'import up from "#up";', + "export default { ok, gone, pad, odd, up };", + ].join("\n"), + "extensions/alias/lib/ok.mjs": 'import pad from "left-pad";\nexport default pad;\n', + "extensions/alias/package.json": { + name: "alias", + version: "1.0.0", + type: "module", + imports: { "#ok": "./lib/ok.mjs", "#gone": "./lib/gone.mjs", "#pad": "left-pad", "#odd": ["./lib/ok.mjs"], "#up": "../x.mjs" }, + }, + }); + const result = checkExtensionModules(path.join(root, "extensions", "alias")); + const errors = result.errors.join("\n"); + assert.ok(result.modules.find((entry) => entry.path === "lib/ok.mjs").reachable); + assert.match(errors, /lib\/ok\.mjs: import "left-pad" is not a Node\.js builtin/); + assert.match(errors, /import "#gone" -> "\.\/lib\/gone\.mjs" references a missing file/); + assert.match(errors, /import "#pad" -> "left-pad" is not a Node\.js builtin/); + assert.match(errors, /import "#odd" uses a package\.json "imports" alias that cannot be analyzed/); + assert.match(errors, /import "#up" uses a package\.json "imports" alias that cannot be analyzed/); +}); + +test("readRegularFile and checkExtensionModules refuse symlinked files", (t) => { + const root = makeRepo({ + "outside/secret.txt": "secret\n", + "extensions/link/extension.mjs": 'import x from "./lib.mjs";\nexport default x;\n', + }); + const extensionDir = path.join(root, "extensions", "link"); + try { + fs.symlinkSync(path.join(root, "outside", "secret.txt"), path.join(extensionDir, "lib.mjs")); + fs.symlinkSync(path.join(root, "outside", "secret.txt"), path.join(extensionDir, "package.json")); + } catch (error) { + if (["EPERM", "EACCES"].includes(error.code)) return t.skip("symlinks unavailable"); + throw error; + } + assert.match(readRegularFile(path.join(extensionDir, "lib.mjs")).error, /symbolic link/); + const errors = checkExtensionModules(extensionDir).errors.join("\n"); + assert.match(errors, /package\.json/); + assert.match(errors, /lib\.mjs/); + assert.doesNotMatch(errors, /secret/); +}); + +test("readRegularFile enforces the size cap and containment root", () => { + const root = makeRepo({ "a/big.json": "x".repeat(64), "b/ok.json": "{}" }); + assert.match(readRegularFile(path.join(root, "a", "big.json"), { maxBytes: 10 }).error, /limit/); + assert.match(readRegularFile(path.join(root, "b", "ok.json"), { root: path.join(root, "a") }).error, /outside/); + assert.equal(readRegularFile(path.join(root, "b", "ok.json"), { root: path.join(root, "b") }).text, "{}"); + assert.equal(readRegularFile(path.join(root, "b", "missing.json")).missing, true); +}); + +test("detectCanvasTargets still validates a plugin whose plugin.json was deleted", async () => { + const root = makeRepo({ + "extensions/orb/extension.mjs": "export {};\n", + "extensions/orb/assets/preview.png": makePng(800, 400), + }); + const targets = detectCanvasTargets(["plugins/orb/plugin.json"], { rootDir: root }); + assert.deepEqual(targets.plugins, ["orb"]); + assert.deepEqual(targets.extensions, ["orb"]); + const report = await runCanvasSmokeTest({ rootDir: root, changedFiles: ["plugins/orb/plugin.json"], install: "never" }); + assert.equal(report.status, "fail"); +}); + +test("detectCanvasTargets uses base plugin manifests when deleted bundle manifests referenced extensions", async () => { + const root = makeRepo({ + "extensions/daily-focus-board/extension.mjs": "export {};\n", + "extensions/daily-focus-board/assets/preview.png": makePng(800, 400), + "plugins/ember/plugin.json": { + $schema: PLUGIN_SCHEMA, + name: "ember", + description: "Ember bundle", + version: "1.0.0", + extensions: { "com.github.awesome-copilot": { extensions: ["./extensions/daily-focus-board"] } }, + }, + }); + git(root, ["init"]); + git(root, ["config", "user.email", "test@example.com"]); + git(root, ["config", "user.name", "Test User"]); + git(root, ["add", "."]); + git(root, ["commit", "-m", "base"]); + const baseRef = git(root, ["rev-parse", "HEAD"]); + fs.rmSync(path.join(root, "plugins", "ember", "plugin.json")); + + const targets = detectCanvasTargets(["plugins/ember/plugin.json"], { rootDir: root, baseRef }); + assert.deepEqual(targets.extensions, ["daily-focus-board"]); + assert.deepEqual(targets.plugins, ["ember"]); + + const report = await runCanvasSmokeTest({ rootDir: root, changedFiles: ["plugins/ember/plugin.json"], baseRef, install: "never" }); + assert.equal(report.status, "fail"); + assert.equal(report.extensions[0].id, "daily-focus-board"); + assert.match(report.plugins[0].errors.join("\n"), /plugins\/ember\/plugin\.json is missing/); +}); + +test("deleted plugin manifest base read errors fail closed when a base ref is provided", async () => { + const root = makeRepo({}); + const report = await runCanvasSmokeTest({ + rootDir: root, + changedFiles: ["plugins/ember/plugin.json"], + baseRef: "missing-base", + install: "never", + }); + assert.equal(report.status, "fail"); + assert.match(report.plugins[0].errors.join("\n"), /cannot read deleted plugins\/ember\/plugin\.json from base missing-base/); +}); + +test("detectCanvasTargets unions base and head extension references for an edited bundle manifest", () => { + const manifest = (refs) => ({ + $schema: PLUGIN_SCHEMA, + name: "ember", + description: "Ember bundle", + version: "1.0.0", + extensions: { "com.github.awesome-copilot": { extensions: refs } }, + }); + const root = makeRepo({ + "extensions/daily-focus-board/extension.mjs": "export {};\n", + "extensions/daily-focus-board/assets/preview.png": makePng(800, 400), + "extensions/orb/extension.mjs": "export {};\n", + "extensions/orb/assets/preview.png": makePng(800, 400), + "plugins/ember/plugin.json": manifest(["./extensions/daily-focus-board"]), + }); + git(root, ["init"]); + git(root, ["config", "user.email", "test@example.com"]); + git(root, ["config", "user.name", "Test User"]); + git(root, ["add", "."]); + git(root, ["commit", "-m", "base"]); + const baseRef = git(root, ["rev-parse", "HEAD"]); + fs.writeFileSync( + path.join(root, "plugins", "ember", "plugin.json"), + `${JSON.stringify(manifest(["./extensions/orb"]), null, 2)}\n`, + ); + + const targets = detectCanvasTargets(["plugins/ember/plugin.json"], { rootDir: root, baseRef }); + assert.deepEqual(targets.extensions, ["daily-focus-board", "orb"]); + assert.deepEqual(targets.plugins, ["ember"]); +}); + +test("the comment stripper treats a slash after a postfix update as division", () => { + const source = [ + 'for (let i = 0; i < n; i++) {}', + 'const r = x++ / y; const dep = require("left-pad");', + 'const s = count-- / total; const dyn = import("./real.mjs");', + 'const t = ++x / y;', + ].join("\n"); + assert.deepEqual(findRequireSpecifiers(source), ["left-pad"]); + assert.deepEqual(findDynamicImportSpecifiers(source), ["./real.mjs"]); + // A prefix update still allows a regex literal to follow. + assert.deepEqual(findRequireSpecifiers('const re = ++i, m = /require("hidden")/;'), []); +}); + +test("unsafe extension references in plugin manifests are rejected and never materialized", async () => { + assert.equal(isSafeExtensionId("orb"), true); + for (const id of ["..", "../x", "a/b", "", ".hidden"]) assert.equal(isSafeExtensionId(id), false, id); + const manifest = { + $schema: PLUGIN_SCHEMA, + name: "evil", + description: "evil", + version: "1.0.0", + extensions: { "com.github.awesome-copilot": { extensions: ["./extensions/../../x", "./extensions/ok"] } }, + }; + assert.deepEqual(unsafeExtensionRefs(manifest), ["./extensions/../../x"]); + const root = makeRepo({ + "plugins/evil/plugin.json": manifest, + "extensions/ok/extension.mjs": "export {};\n", + "extensions/ok/assets/preview.png": makePng(800, 400), + }); + const targets = detectCanvasTargets(["plugins/evil/plugin.json"], { rootDir: root }); + assert.deepEqual(targets.plugins, ["evil"]); + assert.deepEqual(targets.extensions, ["ok"]); + const report = await runCanvasSmokeTest({ rootDir: root, changedFiles: ["plugins/evil/plugin.json"], install: "never" }); + assert.equal(report.status, "fail"); + const pluginErrors = report.plugins.flatMap((plugin) => plugin.errors).join("\n"); + assert.match(pluginErrors, /\.\/extensions\/\.\.\/\.\.\/x/); + assert.notEqual(report.smoke.materialize.evil?.status, "pass"); +}); \ No newline at end of file diff --git a/eng/lib/review-automation-github.mjs b/eng/lib/review-automation-github.mjs new file mode 100644 index 0000000000..27a7d44d43 --- /dev/null +++ b/eng/lib/review-automation-github.mjs @@ -0,0 +1,147 @@ +// Minimal GitHub REST/GraphQL client for maintainer automation scripts. +// +// In workflows it uses GITHUB_TOKEN (or GH_TOKEN) with the global fetch API. +// For local dry runs without a token it falls back to the authenticated `gh` +// CLI (`gh api`), so maintainers never need to export credentials. + +import { spawn } from "node:child_process"; + +const DEFAULT_API_URL = "https://api.github.com"; + +export function createGitHubClient({ + token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN, + apiUrl = process.env.GITHUB_API_URL || DEFAULT_API_URL, + userAgent = "awesome-copilot-maintainer-automation", + transport, +} = {}) { + const baseUrl = apiUrl.replace(/\/$/, ""); + const send = transport ?? (token ? createFetchTransport({ token, userAgent }) : createGhTransport(baseUrl)); + + async function request(method, route, { body, query, allowStatuses = [] } = {}) { + const url = new URL(route.startsWith("http") ? route : `${baseUrl}${route}`); + for (const [key, value] of Object.entries(query ?? {})) { + if (value !== undefined && value !== null) url.searchParams.set(key, String(value)); + } + for (let attempt = 0; ; attempt++) { + const response = await send(method, url, body); + if ((response.status === 502 || response.status === 503) && attempt < 2) { + await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1))); + continue; + } + const data = response.text ? safeJson(response.text) : null; + if ((response.status < 200 || response.status >= 300) && !allowStatuses.includes(response.status)) { + const message = data?.message || response.text || `HTTP ${response.status}`; + const error = new Error(`${method} ${url.pathname} failed: ${response.status} ${message}`); + error.status = response.status; + throw error; + } + return { status: response.status, data, headers: response.headers }; + } + } + + async function paginate(route, { query = {}, itemsKey, maxPages = 20 } = {}) { + const items = []; + let next = null; + let page = 0; + do { + const response = next ? await request("GET", next) : await request("GET", route, { query: { per_page: 100, ...query } }); + const pageItems = itemsKey ? response.data?.[itemsKey] ?? [] : response.data ?? []; + items.push(...pageItems); + next = parseLink(response.headers.get("link"), "next"); + page++; + } while (next && page < maxPages); + return items; + } + + async function graphql(query, variables = {}) { + const response = await request("POST", "/graphql", { body: { query, variables } }); + if (response.data?.errors?.length) { + const error = new Error(`GraphQL request failed: ${response.data.errors.map((item) => item.message).join("; ")}`); + error.errors = response.data.errors; + throw error; + } + return response.data?.data; + } + + return { request, paginate, graphql }; +} + +function createFetchTransport({ token, userAgent }) { + const headers = { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "User-Agent": userAgent, + "X-GitHub-Api-Version": "2022-11-28", + }; + return async (method, url, body) => { + const response = await fetch(url, { + method, + headers: body ? { ...headers, "Content-Type": "application/json" } : headers, + body: body ? JSON.stringify(body) : undefined, + }); + return { status: response.status, text: await response.text(), headers: response.headers }; + }; +} + +function createGhTransport(baseUrl) { + return (method, url, body) => + new Promise((resolve, reject) => { + const endpoint = url.href.startsWith(baseUrl) ? url.href.slice(baseUrl.length + 1) : url.href; + const args = ["api", "--include", "--method", method, endpoint, "-H", "X-GitHub-Api-Version: 2022-11-28"]; + if (body) args.push("--input", "-"); + const child = spawn("gh", args, { stdio: ["pipe", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + child.stderr.setEncoding("utf8").on("data", (chunk) => (stderr += chunk)); + child.on("error", (error) => + reject(new Error(`No GITHUB_TOKEN/GH_TOKEN set and the gh CLI could not be started: ${error.message}`)), + ); + child.on("close", () => { + const parsed = parseIncludedResponse(stdout); + if (!parsed) { + reject(new Error(`gh api ${method} ${endpoint} failed: ${stderr.trim() || "no response"}`)); + return; + } + resolve(parsed); + }); + child.stdin.end(body ? JSON.stringify(body) : undefined); + }); +} + +export function parseIncludedResponse(output) { + const normalized = output.replace(/\r\n/g, "\n"); + const match = normalized.match(/^HTTP\/[\d.]+ (\d{3})[^\n]*\n([\s\S]*?)\n\n([\s\S]*)$/); + if (!match) return null; + const headers = new Headers(); + for (const line of match[2].split("\n")) { + const index = line.indexOf(":"); + if (index > 0) headers.append(line.slice(0, index).trim(), line.slice(index + 1).trim()); + } + return { status: Number(match[1]), text: match[3], headers }; +} + +export function parseLink(linkHeader, rel) { + if (!linkHeader) return null; + for (const part of linkHeader.split(",")) { + const match = part.match(/<([^>]+)>\s*;\s*rel="([^"]+)"/); + if (match && match[2] === rel) return match[1]; + } + return null; +} + +export function parseRepository(value = process.env.GITHUB_REPOSITORY) { + const [owner, repo] = String(value ?? "").split("/"); + if (!owner || !repo) { + throw new Error("Repository must be provided as owner/repo (use --repo or GITHUB_REPOSITORY)."); + } + return { owner, repo }; +} + +function safeJson(text) { + try { + return JSON.parse(text); + } catch { + return null; + } +} diff --git a/eng/review-metrics.mjs b/eng/review-metrics.mjs new file mode 100644 index 0000000000..c0a9ff156e --- /dev/null +++ b/eng/review-metrics.mjs @@ -0,0 +1,669 @@ +#!/usr/bin/env node +// Weekly review operating metrics (github/awesome-copilot#4184, phase 3). +// +// Collects read-only data from the GitHub API, computes review throughput and +// load metrics, and publishes them to a tracking issue. Labels introduced by +// other phases (state and risk tiers) are optional: missing labels are +// reported as "unlabeled" / "unclassified" instead of failing. + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import * as yaml from "js-yaml"; +import { createGitHubClient, parseRepository } from "./lib/review-automation-github.mjs"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +export const ROOT_FOLDER = path.resolve(__dirname, ".."); +export const DEFAULT_CONFIG_PATH = path.join(ROOT_FOLDER, ".github", "review-metrics.yml"); +export const TRACKING_MARKER = ""; +export const REPORT_MARKER = ""; +const HOUR_MS = 60 * 60 * 1000; +const DAY_MS = 24 * HOUR_MS; + +export const DEFAULT_CONFIG = Object.freeze({ + window_days: 7, + targets_business_days: [2, 4], + base_branch: "main", + state_labels: ["awaiting-automation", "requires-submitter-fixes", "ready-for-review", "review-in-progress", "approved"], + risk_labels: ["merge-risk:low", "merge-risk:medium", "merge-risk:high"], + external_plugin_label: "external-plugin", + external_plugin_state_labels: ["awaiting-review", "requires-submitter-fixes", "ready-for-review", "awaiting-approval"], + automation_workflows: [], + tracking_issue: { title: "Review operating metrics", label: "review-metrics" }, +}); + +export function normalizeConfig(raw = {}) { + const input = raw && typeof raw === "object" ? raw : {}; + const config = { ...DEFAULT_CONFIG, ...input, tracking_issue: { ...DEFAULT_CONFIG.tracking_issue, ...(input.tracking_issue ?? {}) } }; + config.window_days = Math.max(1, Number(config.window_days) || DEFAULT_CONFIG.window_days); + config.targets_business_days = (Array.isArray(config.targets_business_days) ? config.targets_business_days : [config.targets_business_days]) + .map(Number) + .filter((value) => Number.isFinite(value) && value > 0) + .sort((a, b) => a - b); + for (const key of ["state_labels", "risk_labels", "external_plugin_state_labels", "automation_workflows"]) { + config[key] = (Array.isArray(config[key]) ? config[key] : []).map(String); + } + return config; +} + +export function loadMetricsConfig(filePath = DEFAULT_CONFIG_PATH) { + if (!fs.existsSync(filePath)) return normalizeConfig({}); + return normalizeConfig(yaml.load(fs.readFileSync(filePath, "utf8")) ?? {}); +} + +// --------------------------------------------------------------------------- +// Statistics helpers (pure) +// --------------------------------------------------------------------------- + +// Elapsed business time in days between two instants. Saturdays and Sundays +// (UTC) contribute nothing; partial weekdays count fractionally. +export function businessDaysBetween(start, end) { + let from = new Date(start).getTime(); + const to = new Date(end).getTime(); + if (!Number.isFinite(from) || !Number.isFinite(to) || to <= from) return 0; + let total = 0; + while (from < to) { + const dayStart = Math.floor(from / DAY_MS) * DAY_MS; + const dayEnd = Math.min(dayStart + DAY_MS, to); + const weekday = new Date(dayStart).getUTCDay(); + if (weekday !== 0 && weekday !== 6) total += (dayEnd - from) / DAY_MS; + from = dayEnd; + } + return total; +} + +// Linear-interpolated percentile (same method as numpy's default). +export function percentile(values, p) { + const sorted = values.filter((value) => Number.isFinite(value)).sort((a, b) => a - b); + if (sorted.length === 0) return null; + const rank = (p / 100) * (sorted.length - 1); + const lower = Math.floor(rank); + const upper = Math.ceil(rank); + return sorted[lower] + (sorted[upper] - sorted[lower]) * (rank - lower); +} + +export function summarizeDurations(hours) { + return { + count: hours.length, + median_hours: roundOrNull(percentile(hours, 50)), + p90_hours: roundOrNull(percentile(hours, 90)), + }; +} + +// Herfindahl-Hirschman index on a 0-10,000 scale plus the top reviewer share. +export function concentration(counts) { + const values = Object.values(counts).filter((value) => value > 0); + const total = values.reduce((sum, value) => sum + value, 0); + if (total === 0) return { total: 0, reviewers: 0, top_share: null, hhi: null, effective_reviewers: null }; + const shares = values.map((value) => value / total); + const hhi = shares.reduce((sum, share) => sum + share * share, 0); + return { + total, + reviewers: values.length, + top_share: round(Math.max(...shares), 3), + hhi: Math.round(hhi * 10000), + effective_reviewers: round(1 / hhi, 2), + }; +} + +function round(value, digits = 1) { + const factor = 10 ** digits; + return Math.round(value * factor) / factor; +} + +function roundOrNull(value, digits = 1) { + return value === null ? null : round(value, digits); +} + +function isBot(actor) { + return !actor || actor.type === "Bot" || /\[bot\]$/i.test(actor.login ?? ""); +} + +// --------------------------------------------------------------------------- +// Metric computation (pure) +// --------------------------------------------------------------------------- + +// First maintainer review submitted at or after the current review clock +// start, so reviews from before a draft/re-ready cycle are not counted. +function firstHumanReview(pr) { + const clockStart = Date.parse(reviewClockStart(pr) ?? ""); + const reviews = (pr.reviews ?? []) + .filter((review) => isMaintainerReview(review, pr) && review.submittedAt && review.state !== "PENDING") + .filter((review) => !Number.isFinite(clockStart) || Date.parse(review.submittedAt) >= clockStart) + .sort((a, b) => Date.parse(a.submittedAt) - Date.parse(b.submittedAt)); + return reviews[0] ?? null; +} + +// Reviews from bots, the PR author, and users without write access do not count. +function isMaintainerReview(review, pr) { + return !isBot(review.author) && review.author?.login !== pr.author?.login && review.authorCanPushToRepository !== false; +} + +function reviewClockStart(pr) { + return pr.readyForReviewAt ?? pr.createdAt; +} + +function tally(items, labelsFor, buckets, fallback) { + const counts = Object.fromEntries([...buckets, fallback].map((bucket) => [bucket, 0])); + for (const item of items) { + const labels = new Set(labelsFor(item).map((label) => label.toLowerCase())); + const matched = buckets.filter((bucket) => labels.has(bucket.toLowerCase())); + if (matched.length === 0) counts[fallback]++; + for (const bucket of matched) counts[bucket]++; + } + return counts; +} + +export function computeMetrics(data, config, { now = new Date() } = {}) { + const nowMs = new Date(now).getTime(); + const windowStart = nowMs - config.window_days * DAY_MS; + const inWindow = (value) => { + const time = Date.parse(value ?? ""); + return Number.isFinite(time) && time >= windowStart && time <= nowMs; + }; + + const contributionPrs = (data.openPrs ?? []).filter((pr) => !isBot(pr.author)); + const readyPrs = contributionPrs.filter((pr) => !pr.isDraft); + const openIssues = data.externalPluginIssues ?? []; + + const open = { + pull_requests: { + total: contributionPrs.length, + drafts: contributionPrs.length - readyPrs.length, + automation_authored_excluded: (data.openPrs ?? []).length - contributionPrs.length, + by_state: tally(readyPrs, (pr) => pr.labels, config.state_labels, "unlabeled"), + by_risk: tally(readyPrs, (pr) => pr.labels, config.risk_labels, "unclassified"), + }, + external_plugin_issues: { + total: openIssues.length, + by_state: tally(openIssues, (issue) => issue.labels, config.external_plugin_state_labels, "unlabeled"), + }, + }; + + // Time to first review: PRs whose first human review landed in the window. + const firstReviewHours = []; + // Time to merge: PRs merged in the window. + const mergeHours = []; + const reviewsByMaintainer = {}; + const prsByMaintainer = {}; + for (const pr of data.windowPrs ?? []) { + if (isBot(pr.author)) continue; + const first = firstHumanReview(pr); + if (first && inWindow(first.submittedAt)) { + firstReviewHours.push((Date.parse(first.submittedAt) - Date.parse(reviewClockStart(pr))) / HOUR_MS); + } + if (pr.mergedAt && inWindow(pr.mergedAt)) { + mergeHours.push((Date.parse(pr.mergedAt) - Date.parse(pr.createdAt)) / HOUR_MS); + } + for (const review of pr.reviews ?? []) { + if (!isMaintainerReview(review, pr) || !inWindow(review.submittedAt)) continue; + const login = review.author.login; + reviewsByMaintainer[login] = (reviewsByMaintainer[login] ?? 0) + 1; + (prsByMaintainer[login] ??= new Set()).add(pr.number); + } + } + + const maintainers = Object.keys(reviewsByMaintainer) + .map((login) => ({ login, reviews: reviewsByMaintainer[login], pull_requests: prsByMaintainer[login].size })) + .sort((a, b) => b.pull_requests - a.pull_requests || b.reviews - a.reviews || a.login.localeCompare(b.login)); + const prCounts = Object.fromEntries(maintainers.map((maintainer) => [maintainer.login, maintainer.pull_requests])); + + // Items waiting on a maintainer beyond the business-day targets. + const waiting = []; + for (const pr of readyPrs) { + const labels = new Set(pr.labels.map((label) => label.toLowerCase())); + if (labels.has("requires-submitter-fixes")) continue; + if (firstHumanReview(pr)) continue; + const since = reviewClockStart(pr); + waiting.push({ kind: "pr", number: pr.number, title: pr.title, url: pr.url, since, business_days: round(businessDaysBetween(since, nowMs), 2) }); + } + for (const issue of openIssues) { + const labels = new Set(issue.labels.map((label) => label.toLowerCase())); + if (!labels.has("ready-for-review") && !labels.has("awaiting-approval")) continue; + const since = issue.readyAt ?? issue.createdAt; + waiting.push({ kind: "issue", number: issue.number, title: issue.title, url: issue.url, since, business_days: round(businessDaysBetween(since, nowMs), 2) }); + } + waiting.sort((a, b) => b.business_days - a.business_days); + const overdue = Object.fromEntries( + config.targets_business_days.map((target) => { + const items = waiting.filter((item) => item.business_days > target); + return [String(target), { count: items.length, items }]; + }), + ); + + // Automation failure rate over completed runs of the review workflows. + const automation = []; + const automationErrors = []; + let failedTotal = 0; + let consideredTotal = 0; + for (const workflow of data.workflowRuns ?? []) { + if (workflow.error) { + automationErrors.push({ workflow: workflow.file, error: workflow.error }); + automation.push({ workflow: workflow.file, found: true, error: workflow.error }); + continue; + } + if (!workflow.found) { + automation.push({ workflow: workflow.file, found: false }); + continue; + } + const runs = workflow.runs.filter((run) => run.status === "completed" && inWindow(run.created_at)); + const considered = runs.filter((run) => !["cancelled", "skipped", "neutral", "action_required", "stale"].includes(run.conclusion)); + const failed = considered.filter((run) => ["failure", "timed_out", "startup_failure"].includes(run.conclusion)); + failedTotal += failed.length; + consideredTotal += considered.length; + automation.push({ + workflow: workflow.file, + name: workflow.name, + found: true, + runs: considered.length, + failed: failed.length, + failure_rate: considered.length ? round(failed.length / considered.length, 3) : null, + }); + } + + return { + generated_at: new Date(nowMs).toISOString(), + window: { days: config.window_days, start: new Date(windowStart).toISOString(), end: new Date(nowMs).toISOString() }, + open, + time_to_first_review: summarizeDurations(firstReviewHours), + time_to_merge: summarizeDurations(mergeHours), + reviews_per_maintainer: maintainers, + reviewer_concentration: { ...concentration(prCounts), basis: "distinct PRs reviewed" }, + overdue, + automation: { + failure_rate: consideredTotal ? round(failedTotal / consideredTotal, 3) : null, + failed: failedTotal, + runs: consideredTotal, + incomplete: automationErrors.length > 0, + errors: automationErrors, + workflows: automation, + }, + }; +} + +// --------------------------------------------------------------------------- +// Rendering +// --------------------------------------------------------------------------- + +function formatHours(hours) { + if (hours === null || hours === undefined) return "n/a"; + if (hours < 48) return `${round(hours, 1)} h`; + return `${round(hours / 24, 1)} d`; +} + +function formatPercent(value) { + return value === null || value === undefined ? "n/a" : `${round(value * 100, 1)}%`; +} + +function escapeCell(value) { + return String(value ?? "").replace(/\|/g, "\\|").replace(/\r?\n/g, " ").replace(/@(?=[A-Za-z0-9])/g, "@\u200b"); +} + +function countsTable(title, counts) { + const lines = [`| ${title} | Count |`, "|---|---:|"]; + for (const [key, value] of Object.entries(counts)) lines.push(`| \`${key}\` | ${value} |`); + return lines.join("\n"); +} + +export function renderReport(metrics, { repository, runUrl } = {}) { + const lines = [REPORT_MARKER, `## Review operating metrics โ€” week ending ${metrics.window.end.slice(0, 10)}`, ""]; + lines.push(`Window: ${metrics.window.start.slice(0, 10)} โ†’ ${metrics.window.end.slice(0, 10)} (${metrics.window.days} days, UTC). Business-day targets skip weekends.`, ""); + + const prs = metrics.open.pull_requests; + lines.push("### Open contributions", ""); + lines.push(`**${prs.total}** open contribution PRs (${prs.drafts} draft, ${prs.automation_authored_excluded} automation-authored excluded) and **${metrics.open.external_plugin_issues.total}** open external plugin submissions.`, ""); + lines.push(countsTable("PR state (non-draft)", prs.by_state), "", countsTable("PR risk tier (non-draft)", prs.by_risk), ""); + lines.push(countsTable("External plugin state", metrics.open.external_plugin_issues.by_state), ""); + + lines.push("### Review speed", "", "| Metric | Samples | Median | p90 |", "|---|---:|---:|---:|"); + lines.push(`| Time to first review | ${metrics.time_to_first_review.count} | ${formatHours(metrics.time_to_first_review.median_hours)} | ${formatHours(metrics.time_to_first_review.p90_hours)} |`); + lines.push(`| Time to merge | ${metrics.time_to_merge.count} | ${formatHours(metrics.time_to_merge.median_hours)} | ${formatHours(metrics.time_to_merge.p90_hours)} |`, ""); + + const conc = metrics.reviewer_concentration; + lines.push("### Reviewer load", ""); + if (metrics.reviews_per_maintainer.length === 0) { + lines.push("No maintainer reviews in this window.", ""); + } else { + lines.push(`Top reviewer share: **${formatPercent(conc.top_share)}** ยท HHI: **${conc.hhi}** (0โ€“10,000; >2,500 = highly concentrated) ยท effective reviewers: **${conc.effective_reviewers}**`, ""); + lines.push("| Maintainer | PRs reviewed | Reviews |", "|---|---:|---:|"); + for (const maintainer of metrics.reviews_per_maintainer) lines.push(`| ${escapeCell(`@${maintainer.login}`)} | ${maintainer.pull_requests} | ${maintainer.reviews} |`); + lines.push(""); + } + + lines.push("### Waiting on maintainers", "", "| Target | Items past target |", "|---|---:|"); + for (const [target, entry] of Object.entries(metrics.overdue)) lines.push(`| ${target} business days | ${entry.count} |`); + const targets = Object.keys(metrics.overdue); + const firstTarget = targets[0]; + if (firstTarget && metrics.overdue[firstTarget].count > 0) { + lines.push("", `
Items past ${firstTarget} business days`, "", "| Item | Waiting (business days) | Title |", "|---|---:|---|"); + for (const item of metrics.overdue[firstTarget].items.slice(0, 50)) { + lines.push(`| ${item.kind === "pr" ? "PR" : "Issue"} [#${item.number}](${item.url}) | ${item.business_days} | ${escapeCell(item.title)} |`); + } + if (metrics.overdue[firstTarget].items.length > 50) lines.push(`| โ€ฆ | | ${metrics.overdue[firstTarget].items.length - 50} more |`); + lines.push("", "
"); + } + lines.push(""); + + const automation = metrics.automation; + lines.push("### Automation health", ""); + lines.push(`Failure rate: **${formatPercent(automation.failure_rate)}${automation.incomplete ? " (incomplete)" : ""}** (${automation.failed} failed of ${automation.runs} completed runs; cancelled/skipped excluded).`, ""); + if (automation.incomplete) lines.push("โš ๏ธ Collection was incomplete because one or more workflow APIs returned errors; affected workflows are excluded from the denominator.", ""); + lines.push("| Workflow | Runs | Failed | Failure rate |", "|---|---:|---:|---:|"); + for (const workflow of automation.workflows) { + if (workflow.error) { + lines.push(`| \`${workflow.workflow}\` | โ€“ | โ€“ | collection error: ${escapeCell(workflow.error)} |`); + } else { + lines.push(workflow.found ? `| \`${workflow.workflow}\` | ${workflow.runs} | ${workflow.failed} | ${formatPercent(workflow.failure_rate)} |` : `| \`${workflow.workflow}\` | โ€“ | โ€“ | not found |`); + } + } + lines.push("", `_Generated ${metrics.generated_at}${repository ? ` for ${repository}` : ""}${runUrl ? ` by [this run](${runUrl})` : ""}. Definitions: docs/maintainers/canvas-evidence-and-metrics.md._`); + return lines.join("\n"); +} + +// --------------------------------------------------------------------------- +// Data collection +// --------------------------------------------------------------------------- + +const PR_FIELDS = ` + id number title url createdAt mergedAt isDraft state + author { login __typename } + labels(first: 50) { nodes { name } } + reviews(first: 100) { pageInfo { hasNextPage endCursor } nodes { state submittedAt authorCanPushToRepository author { login __typename } } } + timelineItems(last: 1, itemTypes: [READY_FOR_REVIEW_EVENT]) { nodes { ... on ReadyForReviewEvent { createdAt } } } +`; + +const REVIEW_FIELDS = "state submittedAt authorCanPushToRepository author { login __typename }"; + +async function searchAll(client, query, fields, maxItems = 1000) { + const items = []; + let cursor = null; + do { + const data = await client.graphql( + `query($q: String!, $cursor: String) { + search(query: $q, type: ISSUE, first: 50, after: $cursor) { + pageInfo { hasNextPage endCursor } + nodes { ... on PullRequest { ${fields} } ... on Issue { ${ISSUE_FIELDS} } } + } + }`, + { q: query, cursor }, + ); + items.push(...(data.search.nodes ?? []).filter(Boolean)); + cursor = data.search.pageInfo.hasNextPage ? data.search.pageInfo.endCursor : null; + } while (cursor && items.length < maxItems); + return items; +} + +async function hydrateReviewPages(client, pullRequests) { + for (const pr of pullRequests) { + let cursor = pr.reviews?.pageInfo?.hasNextPage ? pr.reviews.pageInfo.endCursor : null; + while (cursor) { + const data = await client.graphql( + `query($id: ID!, $cursor: String) { + node(id: $id) { + ... on PullRequest { + reviews(first: 100, after: $cursor) { + pageInfo { hasNextPage endCursor } + nodes { ${REVIEW_FIELDS} } + } + } + } + }`, + { id: pr.id, cursor }, + ); + const page = data.node?.reviews; + if (!page) break; + pr.reviews.nodes.push(...(page.nodes ?? [])); + cursor = page.pageInfo?.hasNextPage ? page.pageInfo.endCursor : null; + } + } + return pullRequests; +} + +async function hydrateIssueLabelEventPages(client, issues) { + for (const issue of issues) { + let cursor = issue.timelineItems?.pageInfo?.hasPreviousPage ? issue.timelineItems.pageInfo.startCursor : null; + while (cursor) { + const data = await client.graphql( + `query($id: ID!, $cursor: String) { + node(id: $id) { + ... on Issue { + timelineItems(last: 100, before: $cursor, itemTypes: [LABELED_EVENT]) { + pageInfo { hasPreviousPage startCursor } + nodes { ... on LabeledEvent { createdAt label { name } } } + } + } + } + }`, + { id: issue.id, cursor }, + ); + const page = data.node?.timelineItems; + if (!page) break; + issue.timelineItems.nodes.unshift(...(page.nodes ?? [])); + cursor = page.pageInfo?.hasPreviousPage ? page.pageInfo.startCursor : null; + } + } + return issues; +} + +const ISSUE_FIELDS = ` + id number title url createdAt + labels(first: 50) { nodes { name } } + timelineItems(last: 100, itemTypes: [LABELED_EVENT]) { + pageInfo { hasPreviousPage startCursor } + nodes { ... on LabeledEvent { createdAt label { name } } } + } +`; + +function normalizePr(node) { + return { + number: node.number, + title: node.title, + url: node.url, + createdAt: node.createdAt, + mergedAt: node.mergedAt, + isDraft: node.isDraft, + readyForReviewAt: node.timelineItems?.nodes?.[0]?.createdAt ?? null, + author: node.author ? { login: node.author.login, type: node.author.__typename } : null, + labels: (node.labels?.nodes ?? []).map((label) => label.name), + reviews: (node.reviews?.nodes ?? []).map((review) => ({ + state: review.state, + submittedAt: review.submittedAt, + authorCanPushToRepository: review.authorCanPushToRepository, + author: review.author ? { login: review.author.login, type: review.author.__typename } : null, + })), + }; +} + +function normalizeIssue(node) { + const readyEvents = (node.timelineItems?.nodes ?? []).filter((event) => ["ready-for-review", "awaiting-approval"].includes(event?.label?.name)); + return { + number: node.number, + title: node.title, + url: node.url, + createdAt: node.createdAt, + readyAt: readyEvents.length ? readyEvents[readyEvents.length - 1].createdAt : null, + labels: (node.labels?.nodes ?? []).map((label) => label.name), + }; +} + +async function fetchWorkflowRuns(client, { owner, repo }, file, since) { + const probe = await client.request("GET", `/repos/${owner}/${repo}/actions/workflows/${encodeURIComponent(file)}`, { allowStatuses: [404] }); + if (probe.status === 404) return { file, found: false, runs: [] }; + const runs = await client.paginate(`/repos/${owner}/${repo}/actions/workflows/${encodeURIComponent(file)}/runs`, { + query: { created: `>=${since.slice(0, 10)}`, exclude_pull_requests: true }, + itemsKey: "workflow_runs", + maxPages: 10, + }); + return { + file, + found: true, + name: probe.data?.name, + runs: runs.map((run) => ({ status: run.status, conclusion: run.conclusion, created_at: run.created_at })), + }; +} + +export async function collectData(client, repository, config, { now = new Date() } = {}) { + const repo = `${repository.owner}/${repository.repo}`; + const since = new Date(new Date(now).getTime() - config.window_days * DAY_MS).toISOString(); + const sinceDate = since.slice(0, 10); + + const openPrNodes = await hydrateReviewPages(client, await searchAll(client, `repo:${repo} is:pr is:open base:${config.base_branch}`, PR_FIELDS)); + const windowPrNodes = await hydrateReviewPages(client, await searchAll(client, `repo:${repo} is:pr base:${config.base_branch} updated:>=${sinceDate}`, PR_FIELDS)); + const openPrs = openPrNodes.map(normalizePr); + const windowPrs = windowPrNodes.map(normalizePr); + let externalPluginIssues = []; + if (config.external_plugin_label) { + const externalPluginIssueNodes = await searchAll( + client, + `repo:${repo} is:issue is:open label:"${config.external_plugin_label}"`, + PR_FIELDS, + ); + externalPluginIssues = (await hydrateIssueLabelEventPages(client, externalPluginIssueNodes)).map(normalizeIssue); + } + const workflowRuns = []; + for (const file of config.automation_workflows) { + try { + workflowRuns.push(await fetchWorkflowRuns(client, repository, file, since)); + } catch (error) { + console.warn(`Could not read runs for ${file}: ${error.message}`); + workflowRuns.push({ file, found: true, runs: [], error: error.message }); + } + } + return { openPrs, windowPrs, externalPluginIssues, workflowRuns }; +} + +// --------------------------------------------------------------------------- +// Publishing +// --------------------------------------------------------------------------- + +async function ensureLabel(client, { owner, repo }, name) { + const existing = await client.request("GET", `/repos/${owner}/${repo}/labels/${encodeURIComponent(name)}`, { allowStatuses: [404] }); + if (existing.status === 404) { + await client.request("POST", `/repos/${owner}/${repo}/labels`, { + body: { name, color: "C5DEF5", description: "Weekly review operating metrics tracking issue" }, + allowStatuses: [422], + }); + } +} + +export function renderTrackingBody(report) { + return [ + TRACKING_MARKER, + "This issue tracks weekly review operating metrics for contribution review (#4184).", + "It is updated automatically by `.github/workflows/review-metrics.yml`; each week's report is also posted as a comment so trends stay visible.", + "", + "---", + "", + report, + ].join("\n"); +} + +export async function publishReport(client, repository, config, report) { + const { owner, repo } = repository; + const label = config.tracking_issue.label; + await ensureLabel(client, repository, label); + const issues = await client.paginate(`/repos/${owner}/${repo}/issues`, { query: { state: "open", labels: label }, maxPages: 5 }); + let issue = issues.find((candidate) => !candidate.pull_request && candidate.body?.includes(TRACKING_MARKER)); + const body = renderTrackingBody(report); + if (issue) { + await client.request("PATCH", `/repos/${owner}/${repo}/issues/${issue.number}`, { body: { body } }); + } else { + issue = (await client.request("POST", `/repos/${owner}/${repo}/issues`, { body: { title: config.tracking_issue.title, body, labels: [label] } })).data; + try { + await client.graphql(`mutation($id: ID!) { pinIssue(input: { issueId: $id }) { issue { number } } }`, { id: issue.node_id }); + } catch (error) { + console.warn(`Could not pin tracking issue #${issue.number}: ${error.message}`); + } + } + await client.request("POST", `/repos/${owner}/${repo}/issues/${issue.number}/comments`, { body: { body: report } }); + return issue; +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +export function parseArgs(argv) { + const options = { + config: DEFAULT_CONFIG_PATH, + repo: process.env.GITHUB_REPOSITORY, + dryRun: false, + outputDir: null, + summaryFile: process.env.GITHUB_STEP_SUMMARY, + now: new Date(), + windowDays: null, + runUrl: process.env.METRICS_RUN_URL || null, + }; + for (let index = 0; index < argv.length; index++) { + const arg = argv[index]; + const next = () => { + const value = argv[++index]; + if (value === undefined) throw new Error(`Missing value for ${arg}`); + return value; + }; + switch (arg) { + case "--config": + options.config = next(); + break; + case "--repo": + options.repo = next(); + break; + case "--dry-run": + options.dryRun = true; + break; + case "--output-dir": + options.outputDir = next(); + break; + case "--summary-file": + options.summaryFile = next(); + break; + case "--now": { + const value = new Date(next()); + if (Number.isNaN(value.getTime())) throw new Error("--now expects an ISO date"); + options.now = value; + break; + } + case "--window-days": + options.windowDays = Number(next()); + if (!Number.isFinite(options.windowDays) || options.windowDays <= 0) throw new Error("--window-days expects a positive number"); + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + return options; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const config = loadMetricsConfig(options.config); + if (options.windowDays) config.window_days = options.windowDays; + const repository = parseRepository(options.repo); + const client = createGitHubClient(); + + const data = await collectData(client, repository, config, { now: options.now }); + const metrics = computeMetrics(data, config, { now: options.now }); + const report = renderReport(metrics, { repository: `${repository.owner}/${repository.repo}`, runUrl: options.runUrl }); + + if (options.outputDir) { + fs.mkdirSync(options.outputDir, { recursive: true }); + fs.writeFileSync(path.join(options.outputDir, "metrics.json"), `${JSON.stringify(metrics, null, 2)}\n`); + fs.writeFileSync(path.join(options.outputDir, "report.md"), `${report}\n`); + } + if (options.summaryFile) fs.appendFileSync(options.summaryFile, `${report}\n`); + + if (options.dryRun) { + if (!options.summaryFile) console.log(report); + console.log("Dry run: tracking issue not updated."); + return; + } + const issue = await publishReport(client, repository, config, report); + console.log(`Published review metrics to ${issue.html_url ?? `#${issue.number}`}`); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/eng/review-metrics.test.mjs b/eng/review-metrics.test.mjs new file mode 100644 index 0000000000..b1ef2a3236 --- /dev/null +++ b/eng/review-metrics.test.mjs @@ -0,0 +1,297 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + businessDaysBetween, + collectData, + computeMetrics, + concentration, + loadMetricsConfig, + normalizeConfig, + parseArgs, + percentile, + renderReport, + renderTrackingBody, + TRACKING_MARKER, +} from "./review-metrics.mjs"; + +test("business days skip weekends and count partial weekdays", () => { + // 2025-01-03 is a Friday. + assert.equal(businessDaysBetween("2025-01-03T00:00:00Z", "2025-01-06T00:00:00Z"), 1); + assert.equal(businessDaysBetween("2025-01-04T00:00:00Z", "2025-01-06T00:00:00Z"), 0); + assert.equal(businessDaysBetween("2025-01-03T12:00:00Z", "2025-01-06T12:00:00Z"), 1); + assert.equal(businessDaysBetween("2025-01-06T00:00:00Z", "2025-01-13T00:00:00Z"), 5); + assert.equal(businessDaysBetween("2025-01-06T00:00:00Z", "2025-01-06T06:00:00Z"), 0.25); + assert.equal(businessDaysBetween("2025-01-06T00:00:00Z", "2025-01-05T00:00:00Z"), 0); +}); + +test("percentiles interpolate linearly and ignore invalid values", () => { + assert.equal(percentile([], 50), null); + assert.equal(percentile([5], 90), 5); + assert.equal(percentile([1, 2, 3, 4], 50), 2.5); + assert.equal(percentile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 90), 9.1); + assert.equal(percentile([3, NaN, 1, 2], 50), 2); +}); + +test("reviewer concentration reports top share and HHI", () => { + assert.deepEqual(concentration({}), { total: 0, reviewers: 0, top_share: null, hhi: null, effective_reviewers: null }); + const single = concentration({ a: 4 }); + assert.equal(single.hhi, 10000); + assert.equal(single.top_share, 1); + const even = concentration({ a: 5, b: 5, c: 5, d: 5 }); + assert.equal(even.hhi, 2500); + assert.equal(even.top_share, 0.25); + assert.equal(even.effective_reviewers, 4); +}); + +const config = normalizeConfig({ automation_workflows: ["a.yml", "missing.yml"] }); +const now = "2025-01-10T12:00:00Z"; // Friday +const maintainer = (login, submittedAt, state = "APPROVED") => ({ state, submittedAt, authorCanPushToRepository: true, author: { login, type: "User" } }); + +function fixture() { + return { + openPrs: [ + { number: 1, title: "Waiting a while", url: "u1", createdAt: "2025-01-03T12:00:00Z", isDraft: false, author: { login: "alice", type: "User" }, labels: ["ready-for-review", "merge-risk:low"], reviews: [] }, + { number: 2, title: "Needs fixes", url: "u2", createdAt: "2025-01-02T12:00:00Z", isDraft: false, author: { login: "bob", type: "User" }, labels: ["requires-submitter-fixes"], reviews: [] }, + { number: 3, title: "Reviewed", url: "u3", createdAt: "2025-01-02T12:00:00Z", isDraft: false, author: { login: "carol", type: "User" }, labels: [], reviews: [maintainer("m1", "2025-01-06T12:00:00Z", "COMMENTED")] }, + { number: 4, title: "Draft", url: "u4", createdAt: "2025-01-09T12:00:00Z", isDraft: true, author: { login: "dave", type: "User" }, labels: [], reviews: [] }, + { number: 5, title: "Bot PR", url: "u5", createdAt: "2025-01-09T12:00:00Z", isDraft: false, author: { login: "github-actions", type: "Bot" }, labels: [], reviews: [] }, + { number: 6, title: "Fresh", url: "u6", createdAt: "2025-01-09T12:00:00Z", isDraft: false, author: { login: "erin", type: "User" }, labels: ["merge-risk:high"], reviews: [] }, + ], + windowPrs: [ + { number: 10, createdAt: "2025-01-06T00:00:00Z", mergedAt: "2025-01-07T00:00:00Z", author: { login: "x", type: "User" }, labels: [], reviews: [maintainer("m1", "2025-01-06T02:00:00Z"), maintainer("m1", "2025-01-06T05:00:00Z")] }, + { number: 11, createdAt: "2025-01-06T00:00:00Z", mergedAt: "2025-01-09T00:00:00Z", author: { login: "y", type: "User" }, labels: [], reviews: [maintainer("m2", "2025-01-06T10:00:00Z"), maintainer("y", "2025-01-06T01:00:00Z"), { ...maintainer("bot", "2025-01-06T00:30:00Z"), author: { login: "copilot", type: "Bot" } }] }, + { number: 12, createdAt: "2024-12-01T00:00:00Z", mergedAt: null, author: { login: "z", type: "User" }, labels: [], reviews: [maintainer("m1", "2024-12-02T00:00:00Z")] }, + { number: 13, createdAt: "2025-01-08T00:00:00Z", mergedAt: null, author: { login: "w", type: "User" }, labels: [], reviews: [{ ...maintainer("drive-by", "2025-01-08T01:00:00Z"), authorCanPushToRepository: false }] }, + ], + externalPluginIssues: [ + { number: 20, title: "Plugin", url: "i20", createdAt: "2024-12-20T00:00:00Z", readyAt: "2025-01-03T00:00:00Z", labels: ["external-plugin", "ready-for-review"] }, + { number: 21, title: "Plugin fixes", url: "i21", createdAt: "2024-12-20T00:00:00Z", labels: ["external-plugin", "requires-submitter-fixes"] }, + ], + workflowRuns: [ + { + file: "a.yml", + found: true, + name: "A", + runs: [ + { status: "completed", conclusion: "success", created_at: "2025-01-08T00:00:00Z" }, + { status: "completed", conclusion: "failure", created_at: "2025-01-08T00:00:00Z" }, + { status: "completed", conclusion: "startup_failure", created_at: "2025-01-08T00:00:00Z" }, + { status: "completed", conclusion: "cancelled", created_at: "2025-01-08T00:00:00Z" }, + { status: "completed", conclusion: "skipped", created_at: "2025-01-08T00:00:00Z" }, + { status: "in_progress", conclusion: null, created_at: "2025-01-08T00:00:00Z" }, + { status: "completed", conclusion: "failure", created_at: "2024-12-01T00:00:00Z" }, + ], + }, + { file: "missing.yml", found: false, runs: [] }, + ], + }; +} + +test("computes open contribution breakdowns with graceful label fallbacks", () => { + const metrics = computeMetrics(fixture(), config, { now }); + const prs = metrics.open.pull_requests; + assert.equal(prs.total, 5, "bot PRs are excluded"); + assert.equal(prs.drafts, 1); + assert.equal(prs.automation_authored_excluded, 1); + assert.equal(prs.by_state["ready-for-review"], 1); + assert.equal(prs.by_state["requires-submitter-fixes"], 1); + assert.equal(prs.by_state.unlabeled, 2); + assert.equal(prs.by_risk["merge-risk:low"], 1); + assert.equal(prs.by_risk["merge-risk:high"], 1); + assert.equal(prs.by_risk.unclassified, 2); + assert.equal(metrics.open.external_plugin_issues.by_state["ready-for-review"], 1); +}); + +test("computes review speed, maintainer load, and concentration", () => { + const metrics = computeMetrics(fixture(), config, { now }); + // First maintainer reviews in window: #10 after 2h, #11 after 10h. Author, bot, + // and non-maintainer reviews (#13) are ignored. + assert.equal(metrics.time_to_first_review.count, 2); + assert.equal(metrics.time_to_first_review.median_hours, 6); + assert.equal(metrics.time_to_merge.count, 2); + assert.equal(metrics.time_to_merge.median_hours, 48); + assert.deepEqual( + metrics.reviews_per_maintainer.map((item) => [item.login, item.pull_requests, item.reviews]), + [["m1", 1, 2], ["m2", 1, 1]], + ); + assert.equal(metrics.reviewer_concentration.top_share, 0.5); + assert.equal(metrics.reviewer_concentration.hhi, 5000); +}); + +test("flags items past business-day targets", () => { + const metrics = computeMetrics(fixture(), config, { now }); + // #1 waits 5 business days, #6 waits 1, issue #20 waits 5.5; #2 needs submitter fixes; #3 was reviewed. + assert.deepEqual(metrics.overdue["2"].items.map((item) => `${item.kind}#${item.number}`), ["issue#20", "pr#1"]); + assert.equal(metrics.overdue["4"].count, 2); +}); + +test("computes automation failure rate over completed, non-cancelled runs", () => { + const metrics = computeMetrics(fixture(), config, { now }); + assert.equal(metrics.automation.runs, 3); + assert.equal(metrics.automation.failed, 2); + assert.equal(metrics.automation.failure_rate, 0.667); + assert.equal(metrics.automation.workflows[1].found, false); +}); + +test("automation API errors are reported as incomplete instead of not found", () => { + const data = fixture(); + data.workflowRuns.push({ file: "error.yml", found: true, runs: [], error: "server unavailable" }); + const metrics = computeMetrics(data, config, { now }); + assert.equal(metrics.automation.incomplete, true); + assert.deepEqual(metrics.automation.errors, [{ workflow: "error.yml", error: "server unavailable" }]); + assert.equal(metrics.automation.runs, 3); + assert.equal(metrics.automation.failed, 2); + assert.equal(metrics.automation.failure_rate, 0.667); + const report = renderReport(metrics, {}); + assert.match(report, /Failure rate: \*\*66\.7% \(incomplete\)\*\*/); + assert.match(report, /`error\.yml` \| โ€“ \| โ€“ \| collection error: server unavailable/); +}); + +test("collectData paginates PR reviews beyond the first 100", async () => { + const pageOneReviews = Array.from({ length: 100 }, (_, index) => maintainer(`m${index}`, "2025-01-06T01:00:00Z")); + const pageTwoReview = maintainer("last", "2025-01-06T02:00:00Z"); + const client = { + graphql: async (query) => { + if (query.includes("search(")) { + return { + search: { + pageInfo: { hasNextPage: false, endCursor: null }, + nodes: [{ + id: "PR_1", + number: 1, + title: "Many reviews", + url: "https://example.test/pull/1", + createdAt: "2025-01-06T00:00:00Z", + mergedAt: null, + isDraft: false, + author: { login: "author", __typename: "User" }, + labels: { nodes: [] }, + reviews: { pageInfo: { hasNextPage: true, endCursor: "page-1" }, nodes: [...pageOneReviews] }, + timelineItems: { nodes: [] }, + }], + }, + }; + } + assert.match(query, /node\(id: \$id\)/); + return { + node: { + reviews: { pageInfo: { hasNextPage: false, endCursor: null }, nodes: [pageTwoReview] }, + }, + }; + }, + request: async () => { throw new Error("request should not be called"); }, + paginate: async () => { throw new Error("paginate should not be called"); }, + }; + const data = await collectData(client, { owner: "o", repo: "r" }, normalizeConfig({ automation_workflows: [], external_plugin_label: "" }), { now }); + assert.equal(data.openPrs[0].reviews.length, 101); + assert.equal(data.windowPrs[0].reviews.length, 101); + assert.equal(data.windowPrs[0].reviews[100].author.login, "last"); +}); + +test("collectData paginates external plugin issue label events", async () => { + let searchCalls = 0; + const client = { + graphql: async (query, variables) => { + if (query.includes("search(")) { + searchCalls++; + if (searchCalls <= 2) { + return { search: { pageInfo: { hasNextPage: false, endCursor: null }, nodes: [] } }; + } + return { + search: { + pageInfo: { hasNextPage: false, endCursor: null }, + nodes: [{ + id: "ISSUE_1", + number: 42, + title: "Long-lived plugin", + url: "https://example.test/issues/42", + createdAt: "2024-01-01T00:00:00Z", + labels: { nodes: [{ name: "external-plugin" }, { name: "ready-for-review" }] }, + timelineItems: { + pageInfo: { hasPreviousPage: true, startCursor: "recent-page" }, + nodes: [{ createdAt: "2025-01-09T00:00:00Z", label: { name: "triage" } }], + }, + }], + }, + }; + } + assert.equal(variables.id, "ISSUE_1"); + assert.equal(variables.cursor, "recent-page"); + return { + node: { + timelineItems: { + pageInfo: { hasPreviousPage: false, startCursor: null }, + nodes: [{ createdAt: "2025-01-03T00:00:00Z", label: { name: "ready-for-review" } }], + }, + }, + }; + }, + request: async () => { throw new Error("request should not be called"); }, + paginate: async () => { throw new Error("paginate should not be called"); }, + }; + const data = await collectData( + client, + { owner: "o", repo: "r" }, + normalizeConfig({ automation_workflows: [], external_plugin_label: "external-plugin" }), + { now }, + ); + assert.equal(data.externalPluginIssues[0].readyAt, "2025-01-03T00:00:00Z"); +}); + +test("handles an empty repository without errors", () => { + const metrics = computeMetrics({}, config, { now }); + assert.equal(metrics.time_to_first_review.median_hours, null); + assert.equal(metrics.automation.failure_rate, null); + const report = renderReport(metrics, {}); + assert.match(report, /No maintainer reviews/); + assert.match(report, /n\/a/); +}); + +test("renders a report and tracking issue body", () => { + const metrics = computeMetrics(fixture(), config, { now }); + const report = renderReport(metrics, { repository: "o/r", runUrl: "https://example.test/run" }); + assert.match(report, /Review operating metrics โ€” week ending 2025-01-10/); + assert.match(report, /Time to first review \| 2 \| 6 h/); + assert.match(report, /HHI: \*\*5000\*\*/); + assert.match(report, /`missing.yml` \| โ€“ \| โ€“ \| not found/); + assert.match(report, /@\u200bm1/, "mentions are neutralized"); + assert.ok(renderTrackingBody(report).startsWith(TRACKING_MARKER)); +}); + +test("config and CLI parsing", () => { + const loaded = loadMetricsConfig(); + assert.equal(loaded.window_days, 7); + assert.deepEqual(loaded.targets_business_days, [2, 4]); + assert.ok(loaded.automation_workflows.includes("canvas-smoke-test.yml")); + assert.equal(loadMetricsConfig("does-not-exist.yml").tracking_issue.label, "review-metrics"); + + const options = parseArgs(["--repo", "o/r", "--dry-run", "--now", "2025-01-10T00:00:00Z", "--window-days", "14"]); + assert.equal(options.dryRun, true); + assert.equal(options.windowDays, 14); + assert.equal(options.now.toISOString(), "2025-01-10T00:00:00.000Z"); + assert.throws(() => parseArgs(["--now", "nope"]), /ISO date/); + assert.throws(() => parseArgs(["--bogus"]), /Unknown argument/); +}); + +test("time to first review ignores reviews submitted before the PR was ready", () => { + const data = { + openPrs: [], + externalPluginIssues: [], + workflowRuns: [], + windowPrs: [ + { + number: 30, + createdAt: "2025-01-06T00:00:00Z", + readyForReviewAt: "2025-01-07T00:00:00Z", + mergedAt: null, + author: { login: "x", type: "User" }, + labels: [], + reviews: [maintainer("m1", "2025-01-06T01:00:00Z", "COMMENTED"), maintainer("m2", "2025-01-07T04:00:00Z")], + }, + ], + }; + const metrics = computeMetrics(data, config, { now }); + assert.equal(metrics.time_to_first_review.count, 1); + assert.equal(metrics.time_to_first_review.median_hours, 4); +}); \ No newline at end of file