From 43189132d4351c1e2b6ea1a45a1dceff001a6c21 Mon Sep 17 00:00:00 2001 From: Marcel Rebro Date: Mon, 22 Jun 2026 13:56:52 +0200 Subject: [PATCH 1/5] docs: add docs-tests harness for Console UI drift detection Docs-as-tests for the Apify Console: extract UI claims (routes, tabs, buttons, headings) from platform docs with an LLM, store them as a reviewed baseline under assertions/, and verify them against Console staging with Playwright. Failures point back to source_file:line. - pages.json: adjustable list of docs pages to cover - scripts/extract*.sh: LLM extraction (run locally, commit the result) - tests/from-doc.spec.ts: evaluate the stored assertions (CI-friendly) - reporters/issues-reporter.ts: machine-readable drift report No secrets committed; auth.json and .env are gitignored. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs-tests/.env.example | 1 + docs-tests/.gitignore | 11 ++ docs-tests/README.md | 124 ++++++++++++++++ docs-tests/package.json | 22 +++ docs-tests/pages.json | 12 ++ docs-tests/playwright.config.ts | 45 ++++++ docs-tests/prompts/assertion-schema.json | 38 +++++ docs-tests/prompts/extract-system.md | 99 +++++++++++++ docs-tests/reporters/issues-reporter.ts | 147 +++++++++++++++++++ docs-tests/scripts/extract-all.sh | 29 ++++ docs-tests/scripts/extract.sh | 73 ++++++++++ docs-tests/tests/auth.setup.ts | 28 ++++ docs-tests/tests/from-doc.spec.ts | 173 +++++++++++++++++++++++ docs-tests/tests/similarity.ts | 60 ++++++++ docs-tests/tsconfig.json | 14 ++ 15 files changed, 876 insertions(+) create mode 100644 docs-tests/.env.example create mode 100644 docs-tests/.gitignore create mode 100644 docs-tests/README.md create mode 100644 docs-tests/package.json create mode 100644 docs-tests/pages.json create mode 100644 docs-tests/playwright.config.ts create mode 100644 docs-tests/prompts/assertion-schema.json create mode 100644 docs-tests/prompts/extract-system.md create mode 100644 docs-tests/reporters/issues-reporter.ts create mode 100755 docs-tests/scripts/extract-all.sh create mode 100755 docs-tests/scripts/extract.sh create mode 100644 docs-tests/tests/auth.setup.ts create mode 100644 docs-tests/tests/from-doc.spec.ts create mode 100644 docs-tests/tests/similarity.ts create mode 100644 docs-tests/tsconfig.json diff --git a/docs-tests/.env.example b/docs-tests/.env.example new file mode 100644 index 0000000000..eb48a3cf80 --- /dev/null +++ b/docs-tests/.env.example @@ -0,0 +1 @@ +CONSOLE_STAGING_URL=https://console.staging-example.apify.dev diff --git a/docs-tests/.gitignore b/docs-tests/.gitignore new file mode 100644 index 0000000000..4b7e1492e1 --- /dev/null +++ b/docs-tests/.gitignore @@ -0,0 +1,11 @@ +# Never commit a logged-in session or local secrets to this public repo. +node_modules/ +.env +auth.json +playwright-report/ +test-results/ +.DS_Store + +# Runtime outputs. The stored assertion baseline (assertions/) IS committed; +# only the per-run report lives here. +output/ diff --git a/docs-tests/README.md b/docs-tests/README.md new file mode 100644 index 0000000000..9c4fe9e25a --- /dev/null +++ b/docs-tests/README.md @@ -0,0 +1,124 @@ +# docs-tests + +Docs-as-tests for the Apify Console. Every UI claim in the platform docs — a +route resolves, a tab is named X, a button exists on page Y — is a testable +assertion. This package extracts those assertions from the docs with an LLM, +stores them as a reviewed baseline, and verifies them against Console staging +with Playwright, so documentation drift is caught automatically. + +``` +pages.json ──extract──▶ assertions/*.json ──Playwright──▶ output/issues.json +(page list) (claude -p) (committed baseline) (vs staging) (drift report) +``` + +## Model + +1. **`pages.json`** is an adjustable list of documentation pages (real source + files under `sources/platform/…`) to cover. +2. **`scripts/extract.sh`** feeds one page to `claude -p` with a strict JSON + schema and writes the result to `assertions/.json`. + `scripts/extract-all.sh` does the whole manifest. +3. **`assertions/`** is the *stored, reviewed baseline* — committed to the repo. + Regenerate it with the LLM whenever docs change, review the diff, commit. + The assertion set is owned by humans even though a model drafts it. +4. **`tests/from-doc.spec.ts`** reads every stored assertion and emits one + Playwright `test()` per assertion, run against `$CONSOLE_STAGING_URL`. +5. Failures point back to `source_file:line` so the offending prose is one click + away, and land in `output/issues.json` for downstream triage. + +The Notion plan *"AI-based testing for docs"* (its Part 1 routes + Part 2 +elements) is the inspiration for which pages and claims to cover — not a fixed +transcription. The authoritative set is whatever the manifest + extractor +produce and a human commits. + +## Assertion kinds + +| Kind | Checks | +| ---------------- | ------------------------------------------------------------------ | +| `route` | Documented path is reachable (HTTP < 400) | +| `element_tab` | Documented tab label exists on the page named in `at` | +| `element_button` | Documented button label exists on the page named in `at` | +| `element_text` | Documented heading/label/field is visible on the page named in `at` | + +## One-time setup + +```bash +pnpm install +pnpm exec playwright install chromium +cp .env.example .env # fill in CONSOLE_STAGING_URL +``` + +## Generate / refresh the assertion baseline + +```bash +# Every page in pages.json: +pnpm extract:all + +# Or a single page: +pnpm extract sources/platform/console/settings.md +``` + +Review the diff in `assertions/`, then commit. This is the step a human owns. + +## Run the tests + +```bash +# Authenticate once (seeded staging user). Opens a browser; log in by hand +# (incl. 2FA), then press the green "Resume" button in Playwright Inspector. +# Writes auth.json (gitignored). Skip on subsequent runs. +pnpm auth + +pnpm test # evaluate all stored assertions against staging +pnpm issues # machine-readable, action-oriented failures +pnpm report # HTML report (failures include screenshots, video, trace) +``` + +`pnpm test` always writes `output/issues.json` — a summary plus one entry per +failing assertion, sorted by `source_line`, each carrying `source_file:line`, +the offending `source_quote`, and a one-line error. For `element_*` failures it +also captures the live page's same-kind labels (`observed_candidates`) and, when +unambiguous, a `suggested_target`, so a downstream LLM can propose a doc fix +without re-running the browser. + +## Adjusting coverage + +Edit `pages.json` and re-run `pnpm extract:all`. Add a page → it gets an +assertion set; remove one → delete its `assertions/.json`. + +## Known gaps (deferred) + +- **Detail-page fixtures.** Assertions about Actor-detail, Schedule-detail, etc. + need a known fixture to navigate to. The runner currently *skips* element + assertions with no `at` route — surfacing the gap without false negatives. + Requires the seeded-user fixtures (1 Actor, 1 task, 1 schedule, named storages, + 1 webhook, 1 completed run) from the Notion plan. +- **Left-nav group check.** The documented global nav items (Dashboard/Store/ + Actors/…) are a Console-wide check, not a per-page claim — not modeled yet. +- **Session-gated pages.** Pages like `/settings/security` re-prompt for + credentials when the stored session is stale; needs a `requires_fresh_session` + field plus a re-auth flow. +- **Multi-step flows.** The schema only supports atomic claims (one + navigate-then-check). "Click X, then Y, then Z" sequences are not modeled. +- **No CI yet.** Local only; wiring into a scheduled GitHub Action (modeled on + `.github/workflows/lychee.yml`) is the follow-up. + +## Files + +``` +docs-tests/ +├── pages.json # adjustable list of docs pages to cover +├── assertions/ # committed baseline, one JSON per page (generated) +├── prompts/ +│ ├── extract-system.md # system prompt + known-routes table +│ └── assertion-schema.json # JSON Schema for the extractor output +├── scripts/ +│ ├── extract.sh # one page → assertions/.json +│ └── extract-all.sh # whole manifest +├── reporters/issues-reporter.ts # custom Playwright reporter → output/issues.json +├── tests/ +│ ├── auth.setup.ts # interactive login, saves auth.json +│ ├── from-doc.spec.ts # reads assertions/*.json, emits tests +│ └── similarity.ts # suggest-replacement helper for failures +├── playwright.config.ts +└── .env # CONSOLE_STAGING_URL (gitignored) +``` diff --git a/docs-tests/package.json b/docs-tests/package.json new file mode 100644 index 0000000000..acdef351d8 --- /dev/null +++ b/docs-tests/package.json @@ -0,0 +1,22 @@ +{ + "name": "docs-tests", + "private": true, + "version": "0.0.1", + "type": "module", + "description": "Docs-as-tests: extract Console-UI claims from sources/platform docs and verify them against staging with Playwright.", + "scripts": { + "extract": "bash scripts/extract.sh", + "extract:all": "bash scripts/extract-all.sh", + "auth": "playwright test --project=setup --headed", + "test": "playwright test --project=tests", + "test:headed": "playwright test --project=tests --headed", + "report": "playwright show-report", + "issues": "jq . output/issues.json" + }, + "devDependencies": { + "@playwright/test": "^1.49.0", + "@types/node": "^22.0.0", + "dotenv": "^16.4.5", + "typescript": "^5.6.0" + } +} diff --git a/docs-tests/pages.json b/docs-tests/pages.json new file mode 100644 index 0000000000..0d968f616c --- /dev/null +++ b/docs-tests/pages.json @@ -0,0 +1,12 @@ +{ + "pages": [ + "sources/platform/console/index.md", + "sources/platform/console/settings.md", + "sources/platform/console/billing.md", + "sources/platform/console/two-factor-authentication.md", + "sources/platform/console/store.md", + "sources/platform/schedules.md", + "sources/platform/collaboration/organization_account/index.md", + "sources/platform/collaboration/organization_account/setup.md" + ] +} diff --git a/docs-tests/playwright.config.ts b/docs-tests/playwright.config.ts new file mode 100644 index 0000000000..f8ad8f3c7b --- /dev/null +++ b/docs-tests/playwright.config.ts @@ -0,0 +1,45 @@ +import { defineConfig, devices } from '@playwright/test'; +import 'dotenv/config'; + +const baseURL = process.env.CONSOLE_STAGING_URL; +if (!baseURL) { + throw new Error( + 'CONSOLE_STAGING_URL is not set. Copy .env.example to .env and fill it in.', + ); +} + +export default defineConfig({ + testDir: './tests', + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: 0, + reporter: [ + ['html', { open: 'never' }], + ['list'], + ['./reporters/issues-reporter.ts'], + ], + + use: { + baseURL, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + + projects: [ + { + name: 'setup', + testMatch: /auth\.setup\.ts/, + use: { ...devices['Desktop Chrome'] }, + }, + { + name: 'tests', + testMatch: /from-doc\.spec\.ts/, + dependencies: ['setup'], + use: { + ...devices['Desktop Chrome'], + storageState: 'auth.json', + }, + }, + ], +}); diff --git a/docs-tests/prompts/assertion-schema.json b/docs-tests/prompts/assertion-schema.json new file mode 100644 index 0000000000..383ac9d8a9 --- /dev/null +++ b/docs-tests/prompts/assertion-schema.json @@ -0,0 +1,38 @@ +{ + "type": "object", + "required": ["source_file", "assertions"], + "properties": { + "source_file": { + "type": "string", + "description": "Path of the input documentation file." + }, + "assertions": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "kind", + "target", + "page_context", + "source_quote", + "source_line", + "needs_auth" + ], + "properties": { + "id": { "type": "string" }, + "kind": { + "type": "string", + "enum": ["route", "element_button", "element_tab", "element_text"] + }, + "target": { "type": "string" }, + "at": { "type": "string" }, + "page_context": { "type": "string" }, + "source_quote": { "type": "string" }, + "source_line": { "type": "integer", "minimum": 1 }, + "needs_auth": { "type": "boolean" } + } + } + } + } +} diff --git a/docs-tests/prompts/extract-system.md b/docs-tests/prompts/extract-system.md new file mode 100644 index 0000000000..ac0ec8f8c5 --- /dev/null +++ b/docs-tests/prompts/extract-system.md @@ -0,0 +1,99 @@ +You are extracting testable UI assertions from Apify Console documentation. + +Your input is a single Markdown documentation page. Your task is to emit a JSON +object describing the testable assertions on that page. + +## Output format — STRICT + +Your ENTIRE response must consist of: +1. The literal string `` on its own line. +2. A single JSON object that conforms to the schema below. +3. The literal string `` on its own line. + +No prose, no commentary, no Markdown fences. Nothing outside the `` tags +will be read. If you have nothing to extract, return +`{"source_file": "...", "assertions": []}`. + +### Schema + +```json +{ + "source_file": "", + "assertions": [ + { + "id": "", + "kind": "route" | "element_button" | "element_tab" | "element_text", + "target": "", + "at": "", + "page_context": "", + "source_quote": "<≤200 char verbatim snippet from the doc>", + "source_line": <1-indexed line number in the input>, + "needs_auth": + } + ] +} +``` + +## Known Apify Console routes + +Use these to populate the `at` field when the doc references a tab, page, or +section that maps to one of these routes. Omit `at` if you can't confidently +map the page_context to a known route. + +| Doc-side label | Console route | +| ------------------------------------ | ------------------------- | +| Settings > Account | `/settings` | +| Settings > Login & Privacy | `/settings/security` | +| Settings > API & Integrations | `/settings/integrations` | +| Settings > Organizations | `/settings/organizations` | +| Settings > Notifications | `/settings/notifications` | +| Billing > Current period | `/billing` | +| Billing > Subscription | `/billing/subscription` | +| Billing > Historical usage | `/billing/historical-usage` | +| Billing > Limits | `/billing#/limits` | +| Actors list | `/actors` | +| Tasks list | `/actors/tasks` | +| Actor templates | `/actors/templates` | +| Actor analytics | `/actors/insights/analytics` | +| Store | `/store` | +| Storage overview | `/storage` | +| Datasets tab | `/storage?tab=datasets` | +| Key-value stores tab | `/storage?tab=keyValueStores` | +| Request queues tab | `/storage?tab=requestQueues` | +| Schedules | `/schedules` | +| Proxy | `/proxy` | +| Proxy groups | `/proxy/groups` | +| Proxy usage | `/proxy/usage` | +| Proxy access | `/proxy/access` | +| Sign-up form | `/sign-up` | +| Sign-in form | `/sign-in` | + +## Extraction rules + +1. **Only extract Console-UI claims.** Skip prose about concepts, payloads, API + behavior, webhooks internals, or anything not visible in the Console. +2. **One assertion = one specific, locatable UI fact.** Examples: + - A specific route is reachable (kind: `route`). + - A specific named button exists on a specific page (kind: `element_button`). + - A specific named tab exists on a specific page (kind: `element_tab`). + - A specific labelled text/field/heading is visible (kind: `element_text`). +3. **`target`** is the exact selector value: + - For `route`: the path (e.g. `/actors`, `/settings/integrations`). No host. + - For `element_*`: the visible label text exactly as it appears in the docs + (preserve casing, drop surrounding backticks/asterisks). +4. **`at`** (optional, for `element_*` only) is the Console path the runner + should navigate to before checking for the element. Pull from the route + table above. Omit if the element isn't on a known landing page (e.g. + detail-page elements that require a fixture). +5. **`page_context`** describes where in the Console the element lives, in plain + English (e.g. "Settings page > Login & Privacy tab"). Even when `at` is set, + include this for human readability. +6. **`source_quote`** is a short verbatim snippet from the doc (≤200 chars) that + justifies the assertion. +7. **`source_line`** is the 1-indexed line number in the input where the quote + appears. +8. **`needs_auth`** is true unless the documented page is `/sign-up` or `/sign-in`. +9. **Be conservative.** If unsure whether the doc claim is locatable as a + selector, omit it. False positives are worse than missed assertions. +10. **`id`** is a short kebab-case slug unique within the output (e.g. + `login-privacy-tab`). diff --git a/docs-tests/reporters/issues-reporter.ts b/docs-tests/reporters/issues-reporter.ts new file mode 100644 index 0000000000..2cf4b24ace --- /dev/null +++ b/docs-tests/reporters/issues-reporter.ts @@ -0,0 +1,147 @@ +import type { Reporter, TestCase, TestResult, FullResult } from '@playwright/test/reporter'; +import { writeFileSync, mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; + +interface AssertionData { + id: string; + kind: string; + target: string; + at?: string; + page_context: string; + source_quote: string; + source_line: number; + source_file: string; +} + +interface ObservedCandidates { + candidate_kind: string; + candidates: string[]; + suggested_target: string | null; +} + +interface Issue { + id: string; + kind: string; + target: string; + at?: string; + page_context: string; + source_file: string; + source_line: number; + source_quote: string; + status: 'failed' | 'timedOut'; + error: string; + duration_ms: number; + suggested_target?: string; + observed_candidates?: { kind: string; values: string[] }; +} + +interface ReportFile { + generated_at: string; + summary: { + total: number; + passed: number; + failed: number; + skipped: number; + pass_rate: string; + }; + issues: Issue[]; +} + +const OUTPUT_PATH = 'output/issues.json'; + +export default class IssuesReporter implements Reporter { + private collected: { test: TestCase; result: TestResult }[] = []; + + onTestEnd(test: TestCase, result: TestResult): void { + // Only collect doc-derived assertions, not the auth setup project. + if (!test.location.file.endsWith('from-doc.spec.ts')) return; + this.collected.push({ test, result }); + } + + async onEnd(_result: FullResult): Promise { + const passed = this.collected.filter(r => r.result.status === 'passed').length; + const failed = this.collected.filter(r => r.result.status === 'failed' || r.result.status === 'timedOut').length; + const skipped = this.collected.filter(r => r.result.status === 'skipped').length; + const total = this.collected.length; + + const issues: Issue[] = this.collected + .filter(r => r.result.status === 'failed' || r.result.status === 'timedOut') + .map(({ test, result }) => { + const data = extractAssertionData(test); + const observed = extractObservedCandidates(test); + const issue: Issue = { + id: data.id, + kind: data.kind, + target: data.target, + at: data.at, + page_context: data.page_context, + source_file: data.source_file, + source_line: data.source_line, + source_quote: data.source_quote, + status: result.status as 'failed' | 'timedOut', + error: summarizeError(result.error?.message), + duration_ms: Math.round(result.duration), + }; + if (observed?.suggested_target) issue.suggested_target = observed.suggested_target; + if (observed?.candidates.length) { + issue.observed_candidates = { kind: observed.candidate_kind, values: observed.candidates }; + } + return issue; + }) + .sort((a, b) => a.source_line - b.source_line); + + const output: ReportFile = { + generated_at: new Date().toISOString(), + summary: { + total, + passed, + failed, + skipped, + pass_rate: total === 0 ? 'n/a' : `${Math.round((passed / total) * 100)}%`, + }, + issues, + }; + + mkdirSync(dirname(OUTPUT_PATH), { recursive: true }); + writeFileSync(OUTPUT_PATH, JSON.stringify(output, null, 2) + '\n'); + // eslint-disable-next-line no-console + console.log(`\nIssues report: ${OUTPUT_PATH} — ${issues.length} issue${issues.length === 1 ? '' : 's'}, ${passed}/${total} passed`); + } +} + +function extractAssertionData(test: TestCase): AssertionData { + const ann = test.annotations.find(a => a.type === 'assertion-data'); + if (ann?.description) { + try { + return JSON.parse(ann.description) as AssertionData; + } catch { + // fall through to placeholder + } + } + return { + id: test.title, + kind: 'unknown', + target: '', + page_context: '', + source_quote: '', + source_line: 0, + source_file: 'unknown', + }; +} + +function extractObservedCandidates(test: TestCase): ObservedCandidates | null { + const ann = test.annotations.find(a => a.type === 'observed-candidates'); + if (!ann?.description) return null; + try { + return JSON.parse(ann.description) as ObservedCandidates; + } catch { + return null; + } +} + +function summarizeError(message: string | undefined): string { + if (!message) return 'no error message'; + const stripped = message.replace(/\x1B\[[0-9;]*[A-Za-z]/g, ''); + const firstLine = stripped.split('\n')[0]!.trim(); + return firstLine.length > 200 ? firstLine.slice(0, 197) + '...' : firstLine; +} diff --git a/docs-tests/scripts/extract-all.sh b/docs-tests/scripts/extract-all.sh new file mode 100755 index 0000000000..04fe0414df --- /dev/null +++ b/docs-tests/scripts/extract-all.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# (Re)extract assertions for every page listed in pages.json. Each page's +# assertions land in assertions/.json — review the diff, commit the set. +# +# Usage: bash scripts/extract-all.sh + +set -euo pipefail + +cd "$(dirname "$0")/.." + +if [[ ! -f pages.json ]]; then + echo "pages.json not found." >&2 + exit 1 +fi + +mapfile -t PAGES < <(jq -r '.pages[]' pages.json) + +echo "Extracting ${#PAGES[@]} page(s) from pages.json..." +for page in "${PAGES[@]}"; do + [[ -z "$page" ]] && continue + echo "" + echo "→ $page" + bash scripts/extract.sh "$page" +done + +echo "" +echo "Done. Stored assertion sets:" +ls -1 assertions/*.json 2>/dev/null || echo " (none)" +jq -s '[.[].assertions | length] | add // 0 | "Total: \(.) assertions across \(input | length) page(s)"' assertions/*.json 2>/dev/null || true diff --git a/docs-tests/scripts/extract.sh b/docs-tests/scripts/extract.sh new file mode 100755 index 0000000000..d618b56ecd --- /dev/null +++ b/docs-tests/scripts/extract.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Extract testable Console-UI assertions from ONE docs page using `claude -p`, +# and write the result to assertions/.json — a committed, reviewed +# baseline the runner later evaluates against staging. +# +# Usage: bash scripts/extract.sh +# e.g. bash scripts/extract.sh sources/platform/console/settings.md +# +# To (re)extract every page in pages.json, use scripts/extract-all.sh. + +set -euo pipefail + +cd "$(dirname "$0")/.." # docs-tests/ +DOCS_TESTS_DIR="$(pwd)" +REPO_ROOT="$(cd .. && pwd)" # apify-docs repo root + +DOC_PATH="${1:?Usage: extract.sh }" +ABS_DOC="$REPO_ROOT/$DOC_PATH" +SYSTEM_PROMPT_FILE="prompts/extract-system.md" + +if [[ ! -f "$ABS_DOC" ]]; then + echo "Doc not found: $DOC_PATH (looked in $ABS_DOC)" >&2 + exit 1 +fi + +# Slug for the stored file: strip the sources/platform/ prefix and .md suffix, +# turn path separators into dashes. e.g. console/settings.md -> console-settings +SLUG=$(printf '%s' "$DOC_PATH" | sed -e 's#^sources/platform/##' -e 's#\.md$##' -e 's#[/ ]#-#g') +OUTPUT_FILE="assertions/$SLUG.json" +mkdir -p assertions + +# Pass the doc inline as the user prompt so source_line numbers refer to the +# real source file's own line numbering. We run claude from /tmp to avoid +# CLAUDE.md/AGENTS.md auto-discovery picking up repo or personal context, and +# disable slash commands so user skills can't inject into the extraction. +USER_PROMPT=$(cat < sentinels; discard any stray prose. +echo "$RAW_OUTPUT" \ + | awk '//{flag=1;next} /<\/output>/{flag=0} flag' \ + > "$OUTPUT_FILE" + +if [[ ! -s "$OUTPUT_FILE" ]]; then + echo "Extraction produced no JSON for $DOC_PATH. Raw model output was:" >&2 + echo "$RAW_OUTPUT" >&2 + rm -f "$OUTPUT_FILE" + exit 1 +fi + +if ! jq empty "$OUTPUT_FILE" 2>/dev/null; then + echo "Extraction produced non-JSON output for $DOC_PATH. Contents:" >&2 + cat "$OUTPUT_FILE" >&2 + exit 1 +fi + +COUNT=$(jq '.assertions | length' "$OUTPUT_FILE") +echo "Wrote $OUTPUT_FILE ($COUNT assertions)" diff --git a/docs-tests/tests/auth.setup.ts b/docs-tests/tests/auth.setup.ts new file mode 100644 index 0000000000..6c0c8b2c7f --- /dev/null +++ b/docs-tests/tests/auth.setup.ts @@ -0,0 +1,28 @@ +import { test as setup, expect } from '@playwright/test'; +import { existsSync } from 'node:fs'; + +const AUTH_FILE = 'auth.json'; + +// Interactive login. Run once with `pnpm auth` (or `npm run auth`). +// The test pauses on the Console sign-in page; you log in by hand +// (including any 2FA), navigate to the home page, then press the +// "Resume" button in Playwright Inspector. Storage state is saved +// to auth.json and reused by the tests project. +// +// Skips if auth.json already exists. Delete the file to re-auth. +setup('authenticate', async ({ page }) => { + if (existsSync(AUTH_FILE) && !process.env.FORCE_REAUTH) { + setup.skip(true, `${AUTH_FILE} already exists; set FORCE_REAUTH=1 to overwrite.`); + return; + } + + await page.goto('/sign-in'); + console.log('\n → Log in by hand. Then press the green "Resume" button in Playwright Inspector.\n'); + await page.pause(); + + // Sanity check: we should now be authenticated. The home redirect lands + // somewhere under /actors, /dashboard, etc. — not back on /sign-in. + await expect(page).not.toHaveURL(/\/sign-in/); + + await page.context().storageState({ path: AUTH_FILE }); +}); diff --git a/docs-tests/tests/from-doc.spec.ts b/docs-tests/tests/from-doc.spec.ts new file mode 100644 index 0000000000..b0bd15490c --- /dev/null +++ b/docs-tests/tests/from-doc.spec.ts @@ -0,0 +1,173 @@ +import { test, expect, type Page } from '@playwright/test'; +import { readFileSync, existsSync, readdirSync } from 'node:fs'; +import { resolve, join } from 'node:path'; +import { suggestReplacement } from './similarity'; + +type AssertionKind = 'route' | 'element_button' | 'element_tab' | 'element_text'; + +interface Assertion { + id: string; + kind: AssertionKind; + target: string; + at?: string; + page_context: string; + source_quote: string; + source_line: number; + needs_auth: boolean; +} + +interface ExtractionOutput { + source_file: string; + assertions: Assertion[]; +} + +// The stored, reviewed baseline: one JSON file per documentation page, produced +// by `pnpm extract:all` and committed to the repo. The runner evaluates every +// stored assertion against staging. +const ASSERTIONS_DIR = resolve(process.cwd(), 'assertions'); + +function loadStoredSets(): ExtractionOutput[] { + if (!existsSync(ASSERTIONS_DIR)) return []; + return readdirSync(ASSERTIONS_DIR) + .filter((f) => f.endsWith('.json')) + .sort() + .map((f) => JSON.parse(readFileSync(join(ASSERTIONS_DIR, f), 'utf8')) as ExtractionOutput); +} + +const sets = loadStoredSets(); + +if (sets.length === 0) { + test('no stored assertions', () => { + test.skip( + true, + `${ASSERTIONS_DIR} is empty. Run \`pnpm extract:all\` to generate the assertion baseline.`, + ); + }); +} + +for (const data of sets) { + test.describe(`Doc-derived assertions from ${data.source_file}`, () => { + for (const a of data.assertions) { + test(`[${a.kind}] ${a.id} — ${a.target}`, async ({ page }) => { + test.info().annotations.push({ + type: 'source', + description: `${data.source_file}:${a.source_line} — "${a.source_quote}"`, + }); + test.info().annotations.push({ type: 'page', description: a.page_context }); + test.info().annotations.push({ + type: 'assertion-data', + description: JSON.stringify({ ...a, source_file: data.source_file }), + }); + + await runAssertion(page, a); + }); + } + }); +} + +async function runAssertion(page: Page, a: Assertion): Promise { + if (a.kind === 'route') { + const resp = await page.goto(a.target, { waitUntil: 'domcontentloaded' }); + expect(resp, `goto ${a.target} returned no response`).not.toBeNull(); + expect(resp!.status(), `goto ${a.target} returned HTTP ${resp!.status()}`).toBeLessThan(400); + return; + } + + // Element assertions: navigate to `at` first if provided. If `at` is missing + // we skip rather than running against an arbitrary page — that's the + // detail-page-fixture problem, not a real assertion failure. + if (!a.at) { + test.skip(true, `assertion ${a.id} has no \`at\` route; needs a fixture`); + return; + } + + await page.goto(a.at, { waitUntil: 'domcontentloaded' }); + + try { + await checkElement(page, a); + } catch (err) { + // On failure, snapshot same-kind elements from the live page so a + // downstream LLM can fix the docs without re-running the browser. The + // reporter reads this annotation into output/issues.json. + await annotateCandidates(page, a); + throw err; + } +} + +async function checkElement(page: Page, a: Assertion): Promise { + switch (a.kind) { + case 'element_button': { + await page.getByRole('button', { name: a.target, exact: false }).first().waitFor({ state: 'visible', timeout: 10_000 }); + return; + } + case 'element_tab': { + // Playwright's `tab` role doesn't always match the Console's tab impl. + // Fall back to a text match if the role lookup misses. + const byRole = page.getByRole('tab', { name: a.target, exact: false }).first(); + const byText = page.getByText(a.target, { exact: false }).first(); + try { + await byRole.waitFor({ state: 'visible', timeout: 5_000 }); + } catch { + await byText.waitFor({ state: 'visible', timeout: 5_000 }); + } + return; + } + case 'element_text': { + await page.getByText(a.target, { exact: false }).first().waitFor({ state: 'visible', timeout: 10_000 }); + return; + } + } +} + +async function annotateCandidates(page: Page, a: Assertion): Promise { + const candidates = await collectCandidates(page, a.kind); + const suggestion = suggestReplacement(a.target, candidates); + test.info().annotations.push({ + type: 'observed-candidates', + description: JSON.stringify({ + candidate_kind: candidateKind(a.kind), + candidates, + suggested_target: suggestion, + }), + }); +} + +async function collectCandidates(page: Page, kind: AssertionKind): Promise { + try { + switch (kind) { + case 'element_button': + return clean(await page.getByRole('button').allTextContents()); + case 'element_tab': + return clean(await page.getByRole('tab').allTextContents()); + case 'element_text': + // Most doc claims of `element_text` are section headings. Capture all + // heading levels so the downstream LLM has the full table of contents. + return clean(await page.locator('h1, h2, h3, h4, h5, h6').allTextContents()); + default: + return []; + } + } catch { + return []; + } +} + +function candidateKind(kind: AssertionKind): string { + switch (kind) { + case 'element_button': return 'button'; + case 'element_tab': return 'tab'; + case 'element_text': return 'heading'; + default: return 'unknown'; + } +} + +function clean(items: string[]): string[] { + const seen = new Set(); + const out: string[] = []; + for (const raw of items) { + const t = raw.replace(/\s+/g, ' ').trim(); + if (!t || seen.has(t)) continue; + seen.add(t); + out.push(t); + } + return out; +} diff --git a/docs-tests/tests/similarity.ts b/docs-tests/tests/similarity.ts new file mode 100644 index 0000000000..234eee3142 --- /dev/null +++ b/docs-tests/tests/similarity.ts @@ -0,0 +1,60 @@ +// Lightweight string-similarity helpers used to suggest a likely replacement +// when an element assertion fails. Deterministic, no deps. + +export function suggestReplacement(target: string, candidates: string[]): string | null { + if (!target || candidates.length === 0) return null; + const ranked = candidates + .map((candidate) => ({ candidate, score: similarity(target, candidate) })) + .sort((a, b) => b.score - a.score); + const top = ranked[0]!; + return top.score >= 0.35 ? top.candidate : null; +} + +export function similarity(a: string, b: string): number { + const aLow = a.trim().toLowerCase(); + const bLow = b.trim().toLowerCase(); + if (!aLow || !bLow) return 0; + if (aLow === bLow) return 1.0; + + // Substring containment (either direction). Common drift: section renamed + // from "Session information" → "Session" (target contains candidate). + if (aLow.includes(bLow) || bLow.includes(aLow)) { + const longer = Math.max(aLow.length, bLow.length); + const shorter = Math.min(aLow.length, bLow.length); + return 0.7 + 0.2 * (shorter / longer); + } + + // Word overlap (Jaccard on whitespace-split tokens). + const wordsA = new Set(aLow.split(/\s+/)); + const wordsB = new Set(bLow.split(/\s+/)); + const intersection = [...wordsA].filter((w) => wordsB.has(w)).length; + const union = wordsA.size + wordsB.size - intersection; + const jaccard = union > 0 ? intersection / union : 0; + if (jaccard > 0) return 0.4 + 0.4 * jaccard; + + // Levenshtein ratio fallback for typo-level changes. + const dist = levenshtein(aLow, bLow); + const maxLen = Math.max(aLow.length, bLow.length); + return Math.max(0, 1 - dist / maxLen) * 0.5; +} + +function levenshtein(a: string, b: string): number { + if (a.length === 0) return b.length; + if (b.length === 0) return a.length; + const dp: number[][] = Array.from({ length: a.length + 1 }, () => + new Array(b.length + 1).fill(0), + ); + for (let i = 0; i <= a.length; i++) dp[i]![0] = i; + for (let j = 0; j <= b.length; j++) dp[0]![j] = j; + for (let i = 1; i <= a.length; i++) { + for (let j = 1; j <= b.length; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + dp[i]![j] = Math.min( + dp[i - 1]![j]! + 1, + dp[i]![j - 1]! + 1, + dp[i - 1]![j - 1]! + cost, + ); + } + } + return dp[a.length]![b.length]!; +} diff --git a/docs-tests/tsconfig.json b/docs-tests/tsconfig.json new file mode 100644 index 0000000000..a76fb0c0e2 --- /dev/null +++ b/docs-tests/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "allowImportingTsExtensions": false, + "noEmit": true + }, + "include": ["tests/**/*", "scripts/**/*", "reporters/**/*", "*.ts"] +} From 9b0078d92a0fe5b7241d921551afca10552296d9 Mon Sep 17 00:00:00 2001 From: Marcel Rebro Date: Mon, 22 Jun 2026 14:14:41 +0200 Subject: [PATCH 2/5] docs: make assertion extraction portable - portable read loop in extract-all.sh (macOS bash 3.2 has no mapfile) - detach claude stdin so it doesn't drain the page list when looped - slice the block with perl so single-line tag+JSON also parses Co-Authored-By: Claude Opus 4.8 (1M context) --- docs-tests/scripts/extract-all.sh | 9 +++++---- docs-tests/scripts/extract.sh | 10 +++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs-tests/scripts/extract-all.sh b/docs-tests/scripts/extract-all.sh index 04fe0414df..bf8ae3c82b 100755 --- a/docs-tests/scripts/extract-all.sh +++ b/docs-tests/scripts/extract-all.sh @@ -13,15 +13,16 @@ if [[ ! -f pages.json ]]; then exit 1 fi -mapfile -t PAGES < <(jq -r '.pages[]' pages.json) +PAGE_COUNT=$(jq -r '.pages | length' pages.json) +echo "Extracting ${PAGE_COUNT} page(s) from pages.json..." -echo "Extracting ${#PAGES[@]} page(s) from pages.json..." -for page in "${PAGES[@]}"; do +# Portable read loop (works on bash 3.2, no mapfile). +while IFS= read -r page; do [[ -z "$page" ]] && continue echo "" echo "→ $page" bash scripts/extract.sh "$page" -done +done < <(jq -r '.pages[]' pages.json) echo "" echo "Done. Stored assertion sets:" diff --git a/docs-tests/scripts/extract.sh b/docs-tests/scripts/extract.sh index d618b56ecd..de7b9a12b7 100755 --- a/docs-tests/scripts/extract.sh +++ b/docs-tests/scripts/extract.sh @@ -44,16 +44,20 @@ $(cat "$ABS_DOC") EOF ) +# ` sentinels; discard any stray prose. -echo "$RAW_OUTPUT" \ - | awk '//{flag=1;next} /<\/output>/{flag=0} flag' \ +# -0777 slurps the whole stream so this works whether the tags and JSON are on +# their own lines or share one line. +printf '%s' "$RAW_OUTPUT" \ + | perl -0777 -ne 'print $1 if m{\s*(.*?)\s*}s' \ > "$OUTPUT_FILE" if [[ ! -s "$OUTPUT_FILE" ]]; then From 87360599d86d602bd4529298a1461037ff50b883 Mon Sep 17 00:00:00 2001 From: Marcel Rebro Date: Mon, 22 Jun 2026 14:14:42 +0200 Subject: [PATCH 3/5] docs: scope pages.json to console test set + add assertion baseline Narrow the manifest to four fixture-free landing pages (console index, settings, billing, store) and commit the AI-extracted baseline: 53 assertions (21 route, 15 text, 11 tab, 6 button). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs-tests/assertions/console-billing.json | 1 + docs-tests/assertions/console-index.json | 1 + docs-tests/assertions/console-settings.json | 1 + docs-tests/assertions/console-store.json | 1 + docs-tests/pages.json | 6 +----- 5 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 docs-tests/assertions/console-billing.json create mode 100644 docs-tests/assertions/console-index.json create mode 100644 docs-tests/assertions/console-settings.json create mode 100644 docs-tests/assertions/console-store.json diff --git a/docs-tests/assertions/console-billing.json b/docs-tests/assertions/console-billing.json new file mode 100644 index 0000000000..c4430fbac2 --- /dev/null +++ b/docs-tests/assertions/console-billing.json @@ -0,0 +1 @@ +{"source_file":"sources/platform/console/billing.md","assertions":[{"id":"billing-route","kind":"route","target":"/billing","page_context":"Billing > Current period page","source_quote":"The **Current period** tab is a comprehensive resource for understanding your platform usage during the ongoing billing cycle.","source_line":11,"needs_auth":true},{"id":"billing-historical-usage-route","kind":"route","target":"/billing/historical-usage","page_context":"Billing > Historical usage page","source_quote":"The **Historical usage** tab provides a detailed view of your monthly platform usage, excluding any free Actor compute units or discounts from your subscription plan.","source_line":21,"needs_auth":true},{"id":"billing-subscription-route","kind":"route","target":"/billing/subscription","page_context":"Billing > Subscription page","source_quote":"Navigate to [Subscription](https://console.apify.com/billing/subscription) section in Apify Console, and click the **Buy add-ons** button to explore the available options.","source_line":53,"needs_auth":true},{"id":"billing-limits-route","kind":"route","target":"/billing#/limits","page_context":"Billing > Limits page","source_quote":"The **Limits** tab displays the usage limits for the Apify platform based on your current subscription plan.","source_line":73,"needs_auth":true},{"id":"current-period-tab","kind":"element_tab","target":"Current period","at":"/billing","page_context":"Billing page tab navigation","source_quote":"The **Current period** tab is a comprehensive resource for understanding your platform usage during the ongoing billing cycle.","source_line":11,"needs_auth":true},{"id":"historical-usage-tab","kind":"element_tab","target":"Historical usage","at":"/billing","page_context":"Billing page tab navigation","source_quote":"The **Historical usage** tab provides a detailed view of your monthly platform usage","source_line":21,"needs_auth":true},{"id":"subscription-tab","kind":"element_tab","target":"Subscription","at":"/billing","page_context":"Billing page tab navigation","source_quote":"The **Subscription** tab offers a central location to manage various aspects of your subscription plan.","source_line":37,"needs_auth":true},{"id":"pricing-tab","kind":"element_tab","target":"Pricing","at":"/billing","page_context":"Billing page tab navigation","source_quote":"The **Pricing** tab offers a way to quickly check all unit pricing for various platform services related to Apify usage for your account.","source_line":61,"needs_auth":true},{"id":"invoices-tab","kind":"element_tab","target":"Invoices","at":"/billing","page_context":"Billing page tab navigation","source_quote":"The **Invoices** tab is where you can find your current and previous invoices for Apify platform usage.","source_line":67,"needs_auth":true},{"id":"limits-tab","kind":"element_tab","target":"Limits","at":"/billing","page_context":"Billing page tab navigation","source_quote":"The **Limits** tab displays the usage limits for the Apify platform based on your current subscription plan.","source_line":73,"needs_auth":true},{"id":"plan-consumption-graph-text","kind":"element_text","target":"Plan Consumption","at":"/billing","page_context":"Billing > Current period tab, Plan Consumption graph","source_quote":"the tab features a **Plan Consumption** Graph. It shows how much of your free or paid plan has been utilized up to this point.","source_line":13,"needs_auth":true},{"id":"actors-statistics-section","kind":"element_text","target":"Actors","at":"/billing","page_context":"Billing > Current period tab, statistics sections under graph","source_quote":"You can access detailed statistics related to **Actors**, **Data transfer**, **Proxy**, and **Storage**.","source_line":15,"needs_auth":true},{"id":"data-transfer-statistics-section","kind":"element_text","target":"Data transfer","at":"/billing","page_context":"Billing > Current period tab, statistics sections under graph","source_quote":"You can access detailed statistics related to **Actors**, **Data transfer**, **Proxy**, and **Storage**.","source_line":15,"needs_auth":true},{"id":"proxy-statistics-section","kind":"element_text","target":"Proxy","at":"/billing","page_context":"Billing > Current period tab, statistics sections under graph","source_quote":"You can access detailed statistics related to **Actors**, **Data transfer**, **Proxy**, and **Storage**.","source_line":15,"needs_auth":true},{"id":"storage-statistics-section","kind":"element_text","target":"Storage","at":"/billing","page_context":"Billing > Current period tab, statistics sections under graph","source_quote":"You can access detailed statistics related to **Actors**, **Data transfer**, **Proxy**, and **Storage**.","source_line":15,"needs_auth":true},{"id":"usage-by-actors-table","kind":"element_text","target":"Usage by Actors","at":"/billing/historical-usage","page_context":"Billing > Historical usage tab, table below the bar chart","source_quote":"Below the bar chart, there is a table titled **Usage by Actors**. This table presents a detailed breakdown of the Compute units used per Actor and the associated costs.","source_line":31,"needs_auth":true},{"id":"buy-add-ons-button","kind":"element_button","target":"Buy add-ons","at":"/billing/subscription","page_context":"Billing > Subscription tab, Plan add-ons section","source_quote":"click the **Buy add-ons** button to explore the available options.","source_line":53,"needs_auth":true}]} \ No newline at end of file diff --git a/docs-tests/assertions/console-index.json b/docs-tests/assertions/console-index.json new file mode 100644 index 0000000000..fc2c7d5e6a --- /dev/null +++ b/docs-tests/assertions/console-index.json @@ -0,0 +1 @@ +{"source_file":"sources/platform/console/index.md","assertions":[{"id":"sign-up-route","kind":"route","target":"/sign-up","page_context":"Apify Console sign-up page","source_quote":"create an account at the [sign-up page](https://console.apify.com/sign-up).","source_line":13,"needs_auth":false},{"id":"sign-in-route","kind":"route","target":"/sign-in","page_context":"Apify Console sign-in page","source_quote":"To sign in to your account, visit the [sign-in page](https://console.apify.com/sign-in).","source_line":42,"needs_auth":false},{"id":"settings-security-route","kind":"route","target":"/settings/security","page_context":"Settings > Login & Privacy section","source_quote":"go to the [Login & Privacy](https://console.apify.com/settings/security) section of your account settings.","source_line":38,"needs_auth":true},{"id":"sign-up-button","kind":"element_button","target":"Sign up","at":"/sign-up","page_context":"Sign-up form page","source_quote":"After you click the **Sign up** button, we will send you a verification email.","source_line":19,"needs_auth":false},{"id":"resend-verification-email-button","kind":"element_button","target":"Resend verification email","page_context":"Email verification page reached after sign-up flow","source_quote":"On the verification page, you can click on the **Resend verification email** button to send the email again.","source_line":25,"needs_auth":true},{"id":"continue-with-google-button","kind":"element_button","target":"Continue with Google","at":"/sign-up","page_context":"Sign-up form page","source_quote":"click the **Continue with Google** or **Continue with GitHub** buttons.","source_line":29,"needs_auth":false},{"id":"continue-with-github-button","kind":"element_button","target":"Continue with GitHub","at":"/sign-up","page_context":"Sign-up form page","source_quote":"click the **Continue with Google** or **Continue with GitHub** buttons.","source_line":29,"needs_auth":false},{"id":"recently-viewed-text","kind":"element_text","target":"Recently viewed","page_context":"Dashboard section shown after sign-in","source_quote":"**Recently viewed**: Displays Actors you have recently accessed.","source_line":48,"needs_auth":true},{"id":"suggested-actors-for-you-text","kind":"element_text","target":"Suggested Actors for you","page_context":"Dashboard section shown after sign-in","source_quote":"**Suggested Actors for you**: Recommends Actors that might interest you based on your and other users' recent activities.","source_line":50,"needs_auth":true},{"id":"actor-runs-text","kind":"element_text","target":"Actor runs","page_context":"Dashboard section shown after sign-in","source_quote":"**Actor runs**: Shows your recent Actor runs, as well as scheduled runs and tasks.","source_line":52,"needs_auth":true},{"id":"store-route","kind":"route","target":"/store","page_context":"Apify Store section accessible from left-side panel","source_quote":"| [Apify Store](/platform/console/store) | G + O | Search for Actors that suit your web-scraping needs. |","source_line":62,"needs_auth":true},{"id":"actors-route","kind":"route","target":"/actors","page_context":"Actors section accessible from left-side panel","source_quote":"| [Actors](/platform/actors) | G + A | View recent and bookmarked Actors. |","source_line":64,"needs_auth":true},{"id":"tasks-route","kind":"route","target":"/actors/tasks","page_context":"Saved tasks section accessible from left-side panel","source_quote":"| [Saved tasks](/platform/actors/running/tasks) | G + T | View your saved tasks. |","source_line":66,"needs_auth":true},{"id":"schedules-route","kind":"route","target":"/schedules","page_context":"Schedules section accessible from left-side panel","source_quote":"| [Schedules](/platform/schedules) | G + U | Schedule Actor runs and tasks to run at a specified time. |","source_line":68,"needs_auth":true},{"id":"proxy-route","kind":"route","target":"/proxy","page_context":"Proxy section accessible from left-side panel","source_quote":"| [Proxy](/platform/proxy) | G + P | View your proxy usage and credentials. |","source_line":70,"needs_auth":true},{"id":"storage-route","kind":"route","target":"/storage","page_context":"Storage section accessible from left-side panel","source_quote":"| [Storage](/platform/storage) | G + E | View stored results of your runs in various data formats. |","source_line":71,"needs_auth":true},{"id":"billing-route","kind":"route","target":"/billing","page_context":"Billing section accessible from left-side panel","source_quote":"| [Billing](/platform/console/billing) | G + B | View billing information, statistics, and invoices. |","source_line":72,"needs_auth":true},{"id":"settings-route","kind":"route","target":"/settings","page_context":"Settings section accessible from left-side panel","source_quote":"| [Settings](/platform/console/settings) | G + S | Change settings of your account. |","source_line":73,"needs_auth":true}]} \ No newline at end of file diff --git a/docs-tests/assertions/console-settings.json b/docs-tests/assertions/console-settings.json new file mode 100644 index 0000000000..312e0386b5 --- /dev/null +++ b/docs-tests/assertions/console-settings.json @@ -0,0 +1 @@ +{"source_file":"sources/platform/console/settings.md","assertions":[{"id":"settings-route","kind":"route","target":"/settings","page_context":"Settings > Account page","source_quote":"By clicking the **Settings** tab on the side menu, you will be presented with an Account page where you can view and edit various settings regarding your account","source_line":11,"needs_auth":true},{"id":"login-privacy-route","kind":"route","target":"/settings/security","page_context":"Settings > Login & Privacy tab","source_quote":"The **Login & Privacy** tab (**Security & Privacy** for organization accounts) contains sensitive settings related to authentication and session management.","source_line":21,"needs_auth":true},{"id":"api-integrations-route","kind":"route","target":"/settings/integrations","page_context":"Settings > API & Integrations tab","source_quote":"The **API & Integrations** tab provides essential tools for accessing the Apify platform programmatically.","source_line":39,"needs_auth":true},{"id":"organizations-route","kind":"route","target":"/settings/organizations","page_context":"Settings > Organizations tab","source_quote":"The **Organizations** tab is where you can view your accounts' current organizations, create new organizations, or convert your user account into an organization account.","source_line":97,"needs_auth":true},{"id":"notifications-route","kind":"route","target":"/settings/notifications","page_context":"Settings > Notifications tab","source_quote":"The **Notifications** tab allows you to customize your notification preferences.","source_line":101,"needs_auth":true},{"id":"login-privacy-tab","kind":"element_tab","target":"Login & Privacy","at":"/settings/security","page_context":"Settings sidebar navigation, Login & Privacy tab","source_quote":"The **Login & Privacy** tab (**Security & Privacy** for organization accounts) contains sensitive settings related to authentication and session management.","source_line":21,"needs_auth":true},{"id":"session-information-section","kind":"element_text","target":"Session information","at":"/settings/security","page_context":"Settings > Login & Privacy tab, Session information section heading","source_quote":"In the **Session information** section, you can adjust the session configuration. You can modify the default session lifespan of 90 days.","source_line":35,"needs_auth":true},{"id":"api-integrations-tab","kind":"element_tab","target":"API & Integrations","at":"/settings/integrations","page_context":"Settings sidebar navigation, API & Integrations tab","source_quote":"The **API & Integrations** tab provides essential tools for accessing the Apify platform programmatically.","source_line":39,"needs_auth":true},{"id":"mcp-connectors-section","kind":"element_text","target":"MCP connectors","at":"/settings/integrations","page_context":"Settings > API & Integrations tab, MCP connectors section heading","source_quote":"The **MCP connectors** section lets you authorize third-party MCP servers (such as Notion, Slack, GitHub, or Supabase) once and reuse those connections across any Actor that accepts them.","source_line":43,"needs_auth":true},{"id":"create-new-connector-button","kind":"element_button","target":"Create new connector","at":"/settings/integrations","page_context":"Settings > API & Integrations tab, MCP connectors section","source_quote":"Open **Settings > API & Integrations > MCP connectors** and click **Create new connector**.","source_line":47,"needs_auth":true},{"id":"organizations-tab","kind":"element_tab","target":"Organizations","at":"/settings/organizations","page_context":"Settings sidebar navigation, Organizations tab","source_quote":"The **Organizations** tab is where you can view your accounts' current organizations, create new organizations, or convert your user account into an organization account.","source_line":97,"needs_auth":true},{"id":"notifications-tab","kind":"element_tab","target":"Notifications","at":"/settings/notifications","page_context":"Settings sidebar navigation, Notifications tab","source_quote":"The **Notifications** tab allows you to customize your notification preferences. Here, you can specify the types of updates you wish to receive and select the methods by which you receive them.","source_line":101,"needs_auth":true},{"id":"referrals-tab","kind":"element_tab","target":"Referrals","page_context":"Settings sidebar navigation, Referrals tab","source_quote":"The **Referrals** tab lets you share Apify with others and earn rewards. You can find your referral link and track the status of your referrals.","source_line":105,"needs_auth":true}]} \ No newline at end of file diff --git a/docs-tests/assertions/console-store.json b/docs-tests/assertions/console-store.json new file mode 100644 index 0000000000..9f2bd9fa3d --- /dev/null +++ b/docs-tests/assertions/console-store.json @@ -0,0 +1 @@ +{"source_file": "sources/platform/console/store.md", "assertions": [{"id": "store-route", "kind": "route", "target": "/store", "page_context": "Apify Store main page", "source_quote": "[Apify Store](https://apify.com/store) is a place where you can explore a variety of Actors, both created and maintained by Apify or the community.", "source_line": 9, "needs_auth": true}, {"id": "store-category-filter", "kind": "element_text", "target": "Category", "at": "/store", "page_context": "Apify Store page – results filter/sort criteria", "source_quote": "You can also organize the results from the store by different criteria, including:\n\n* Category", "source_line": 14, "needs_auth": true}, {"id": "store-pricing-model-filter", "kind": "element_text", "target": "Pricing model", "at": "/store", "page_context": "Apify Store page – results filter/sort criteria", "source_quote": "* Pricing model", "source_line": 15, "needs_auth": true}, {"id": "store-developers-filter", "kind": "element_text", "target": "Developers", "at": "/store", "page_context": "Apify Store page – results filter/sort criteria", "source_quote": "* Developers", "source_line": 16, "needs_auth": true}, {"id": "store-relevance-filter", "kind": "element_text", "target": "Relevance", "at": "/store", "page_context": "Apify Store page – results filter/sort criteria", "source_quote": "* Relevance", "source_line": 17, "needs_auth": true}]} \ No newline at end of file diff --git a/docs-tests/pages.json b/docs-tests/pages.json index 0d968f616c..f06479bf16 100644 --- a/docs-tests/pages.json +++ b/docs-tests/pages.json @@ -3,10 +3,6 @@ "sources/platform/console/index.md", "sources/platform/console/settings.md", "sources/platform/console/billing.md", - "sources/platform/console/two-factor-authentication.md", - "sources/platform/console/store.md", - "sources/platform/schedules.md", - "sources/platform/collaboration/organization_account/index.md", - "sources/platform/collaboration/organization_account/setup.md" + "sources/platform/console/store.md" ] } From 4d9a903666eee1418030fc856daf83c523a3a695 Mon Sep 17 00:00:00 2001 From: Marcel Rebro Date: Mon, 22 Jun 2026 17:01:28 +0200 Subject: [PATCH 4/5] docs: authenticate via in-memory worker fixture (no auth file, no 2FA) Replace the interactive auth.setup.ts + auth.json storageState handoff with a worker-scoped fixture that logs in fresh each run from CONSOLE_STAGING_USER_EMAIL/_PASSWORD (.env locally, GitHub Secrets in CI) and keeps the session in memory. Nothing is written to or read from disk, so no auth file has to pre-exist in the GitHub Action. Seeded user has no 2FA, so it's a plain email+password submit; drop the setup project and pnpm auth. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs-tests/README.md | 17 ++++----- docs-tests/package.json | 1 - docs-tests/playwright.config.ts | 15 +++----- docs-tests/tests/auth.fixture.ts | 59 +++++++++++++++++++++++++++++++ docs-tests/tests/auth.setup.ts | 28 --------------- docs-tests/tests/from-doc.spec.ts | 3 +- 6 files changed, 75 insertions(+), 48 deletions(-) create mode 100644 docs-tests/tests/auth.fixture.ts delete mode 100644 docs-tests/tests/auth.setup.ts diff --git a/docs-tests/README.md b/docs-tests/README.md index 9c4fe9e25a..ec49956648 100644 --- a/docs-tests/README.md +++ b/docs-tests/README.md @@ -45,7 +45,7 @@ produce and a human commits. ```bash pnpm install pnpm exec playwright install chromium -cp .env.example .env # fill in CONSOLE_STAGING_URL +cp .env.example .env # fill in CONSOLE_STAGING_URL + seeded-user email/password ``` ## Generate / refresh the assertion baseline @@ -63,16 +63,17 @@ Review the diff in `assertions/`, then commit. This is the step a human owns. ## Run the tests ```bash -# Authenticate once (seeded staging user). Opens a browser; log in by hand -# (incl. 2FA), then press the green "Resume" button in Playwright Inspector. -# Writes auth.json (gitignored). Skip on subsequent runs. -pnpm auth - pnpm test # evaluate all stored assertions against staging pnpm issues # machine-readable, action-oriented failures pnpm report # HTML report (failures include screenshots, video, trace) ``` +Authentication is automatic: a worker-scoped fixture (`tests/auth.fixture.ts`) +logs in once per run with `CONSOLE_STAGING_USER_EMAIL` / `_PASSWORD` and keeps +the session in memory. **No `auth.json` is written or read** — nothing has to +pre-exist, so it behaves identically locally and in CI (where the credentials +come from GitHub Secrets). The seeded staging user has no 2FA. + `pnpm test` always writes `output/issues.json` — a summary plus one entry per failing assertion, sorted by `source_line`, each carrying `source_file:line`, the offending `source_quote`, and a one-line error. For `element_*` failures it @@ -95,7 +96,7 @@ assertion set; remove one → delete its `assertions/.json`. - **Left-nav group check.** The documented global nav items (Dashboard/Store/ Actors/…) are a Console-wide check, not a per-page claim — not modeled yet. - **Session-gated pages.** Pages like `/settings/security` re-prompt for - credentials when the stored session is stale; needs a `requires_fresh_session` + credentials even within a valid session; needs a `requires_fresh_session` field plus a re-auth flow. - **Multi-step flows.** The schema only supports atomic claims (one navigate-then-check). "Click X, then Y, then Z" sequences are not modeled. @@ -116,7 +117,7 @@ docs-tests/ │ └── extract-all.sh # whole manifest ├── reporters/issues-reporter.ts # custom Playwright reporter → output/issues.json ├── tests/ -│ ├── auth.setup.ts # interactive login, saves auth.json +│ ├── auth.fixture.ts # worker-scoped login from env creds (in-memory session) │ ├── from-doc.spec.ts # reads assertions/*.json, emits tests │ └── similarity.ts # suggest-replacement helper for failures ├── playwright.config.ts diff --git a/docs-tests/package.json b/docs-tests/package.json index acdef351d8..c4c14ac3fc 100644 --- a/docs-tests/package.json +++ b/docs-tests/package.json @@ -7,7 +7,6 @@ "scripts": { "extract": "bash scripts/extract.sh", "extract:all": "bash scripts/extract-all.sh", - "auth": "playwright test --project=setup --headed", "test": "playwright test --project=tests", "test:headed": "playwright test --project=tests --headed", "report": "playwright show-report", diff --git a/docs-tests/playwright.config.ts b/docs-tests/playwright.config.ts index f8ad8f3c7b..eac65084bc 100644 --- a/docs-tests/playwright.config.ts +++ b/docs-tests/playwright.config.ts @@ -26,20 +26,15 @@ export default defineConfig({ video: 'retain-on-failure', }, + // No setup project and no storageState file: authentication is a worker-scoped + // fixture (tests/auth.fixture.ts) that logs in fresh each run and keeps the + // session in memory. Nothing on disk has to pre-exist — works the same locally + // and in CI, where credentials come from GitHub Secrets. projects: [ - { - name: 'setup', - testMatch: /auth\.setup\.ts/, - use: { ...devices['Desktop Chrome'] }, - }, { name: 'tests', testMatch: /from-doc\.spec\.ts/, - dependencies: ['setup'], - use: { - ...devices['Desktop Chrome'], - storageState: 'auth.json', - }, + use: { ...devices['Desktop Chrome'] }, }, ], }); diff --git a/docs-tests/tests/auth.fixture.ts b/docs-tests/tests/auth.fixture.ts new file mode 100644 index 0000000000..b628c9c715 --- /dev/null +++ b/docs-tests/tests/auth.fixture.ts @@ -0,0 +1,59 @@ +import { test as base, type BrowserContext } from '@playwright/test'; +import 'dotenv/config'; + +const EMAIL = process.env.CONSOLE_STAGING_USER_EMAIL; +const PASSWORD = process.env.CONSOLE_STAGING_USER_PASSWORD; +const BASE_URL = process.env.CONSOLE_STAGING_URL; + +type StorageState = Awaited>; + +// Worker-scoped authentication. Logs in ONCE per worker with credentials from +// the environment (.env locally, GitHub Secrets in CI) and hands the resulting +// session to every test as an in-memory storageState. +// +// No auth.json is ever written or read — nothing has to pre-exist in CI; each +// run authenticates fresh. The seeded staging user has no 2FA, so this is a +// plain email + password form submit. +export const test = base.extend({ + authState: [ + async ({ browser }, use) => { + if (!EMAIL || !PASSWORD) { + throw new Error( + 'CONSOLE_STAGING_USER_EMAIL and CONSOLE_STAGING_USER_PASSWORD must be set ' + + '(.env locally, GitHub Secrets in CI).', + ); + } + + const context = await browser.newContext({ baseURL: BASE_URL }); + const page = await context.newPage(); + await page.goto('/sign-in'); + + // Selectors aren't pinned to the Console DOM; each field tries a few + // resilient locators. If a step misses, the retained trace shows the form. + const email = page + .locator('input[type="email"], input[name="email"], input[name="username"]') + .first(); + await email.waitFor({ state: 'visible', timeout: 15_000 }); + await email.fill(EMAIL); + + const password = page.locator('input[type="password"], input[name="password"]').first(); + await password.fill(PASSWORD); + + await page.getByRole('button', { name: /sign in|log in|continue/i }).first().click(); + + // Confirm we left the sign-in page; otherwise creds/selectors are wrong. + await page.waitForURL((url) => !/\/sign-in/.test(url.pathname), { timeout: 15_000 }); + + const state = await context.storageState(); + await context.close(); + + await use(state); + }, + { scope: 'worker' }, + ], + + // Feed the in-memory session to every test's context. + storageState: ({ authState }, use) => use(authState), +}); + +export { expect } from '@playwright/test'; diff --git a/docs-tests/tests/auth.setup.ts b/docs-tests/tests/auth.setup.ts deleted file mode 100644 index 6c0c8b2c7f..0000000000 --- a/docs-tests/tests/auth.setup.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { test as setup, expect } from '@playwright/test'; -import { existsSync } from 'node:fs'; - -const AUTH_FILE = 'auth.json'; - -// Interactive login. Run once with `pnpm auth` (or `npm run auth`). -// The test pauses on the Console sign-in page; you log in by hand -// (including any 2FA), navigate to the home page, then press the -// "Resume" button in Playwright Inspector. Storage state is saved -// to auth.json and reused by the tests project. -// -// Skips if auth.json already exists. Delete the file to re-auth. -setup('authenticate', async ({ page }) => { - if (existsSync(AUTH_FILE) && !process.env.FORCE_REAUTH) { - setup.skip(true, `${AUTH_FILE} already exists; set FORCE_REAUTH=1 to overwrite.`); - return; - } - - await page.goto('/sign-in'); - console.log('\n → Log in by hand. Then press the green "Resume" button in Playwright Inspector.\n'); - await page.pause(); - - // Sanity check: we should now be authenticated. The home redirect lands - // somewhere under /actors, /dashboard, etc. — not back on /sign-in. - await expect(page).not.toHaveURL(/\/sign-in/); - - await page.context().storageState({ path: AUTH_FILE }); -}); diff --git a/docs-tests/tests/from-doc.spec.ts b/docs-tests/tests/from-doc.spec.ts index b0bd15490c..0bd4bda86b 100644 --- a/docs-tests/tests/from-doc.spec.ts +++ b/docs-tests/tests/from-doc.spec.ts @@ -1,4 +1,5 @@ -import { test, expect, type Page } from '@playwright/test'; +import { type Page } from '@playwright/test'; +import { test, expect } from './auth.fixture'; import { readFileSync, existsSync, readdirSync } from 'node:fs'; import { resolve, join } from 'node:path'; import { suggestReplacement } from './similarity'; From 93c4ef75c3d12989c0c4476e766f3974134ef34b Mon Sep 17 00:00:00 2001 From: Marcel Rebro Date: Mon, 22 Jun 2026 17:47:42 +0200 Subject: [PATCH 5/5] docs: fix Console login flow + add step logging to docs-tests harness The worker auth fixture assumed a single combined sign-in form; Apify's sign-in is two-step (email -> Next -> password -> Log in), both steps on /sign-in. Pin the real selectors, avoid the SSO buttons, and wait on domcontentloaded (the Console SPA never reaches networkidle). Add timestamped step logging to the login fixture so a slow or stuck login is visible instead of a silent hang before any test reports. Also adds the pnpm workspace + lockfile so docs-tests installs in isolation, and documents the staging-user vars in .env.example. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs-tests/.env.example | 7 +++ docs-tests/pnpm-lock.yaml | 86 ++++++++++++++++++++++++++++++++ docs-tests/pnpm-workspace.yaml | 5 ++ docs-tests/tests/auth.fixture.ts | 49 +++++++++++++++--- 4 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 docs-tests/pnpm-lock.yaml create mode 100644 docs-tests/pnpm-workspace.yaml diff --git a/docs-tests/.env.example b/docs-tests/.env.example index eb48a3cf80..765e55d827 100644 --- a/docs-tests/.env.example +++ b/docs-tests/.env.example @@ -1 +1,8 @@ +# Staging Console base URL (required). CONSOLE_STAGING_URL=https://console.staging-example.apify.dev + +# Seeded staging user (required to run the tests). Used by tests/auth.fixture.ts +# to log in fresh each run. Locally these live here; in CI they come from GitHub +# Secrets, never from a committed file. The seeded account has no 2FA. +CONSOLE_STAGING_USER_EMAIL= +CONSOLE_STAGING_USER_PASSWORD= \ No newline at end of file diff --git a/docs-tests/pnpm-lock.yaml b/docs-tests/pnpm-lock.yaml new file mode 100644 index 0000000000..025973cf3c --- /dev/null +++ b/docs-tests/pnpm-lock.yaml @@ -0,0 +1,86 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + '@playwright/test': + specifier: ^1.49.0 + version: 1.61.0 + '@types/node': + specifier: ^22.0.0 + version: 22.20.0 + dotenv: + specifier: ^16.4.5 + version: 16.6.1 + typescript: + specifier: ^5.6.0 + version: 5.9.3 + +packages: + + '@playwright/test@1.61.0': + resolution: {integrity: sha512-cKA5B6lpFEMyMGjxF54QihfYpB4FkEGH+qZhtArDEG+wezQAJY8Pq6C7T1SjWz+FFzt3TbyoXBQYk/0292TdJA==} + engines: {node: '>=18'} + hasBin: true + + '@types/node@22.20.0': + resolution: {integrity: sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g==} + + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + playwright-core@1.61.0: + resolution: {integrity: sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.61.0: + resolution: {integrity: sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==} + engines: {node: '>=18'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + +snapshots: + + '@playwright/test@1.61.0': + dependencies: + playwright: 1.61.0 + + '@types/node@22.20.0': + dependencies: + undici-types: 6.21.0 + + dotenv@16.6.1: {} + + fsevents@2.3.2: + optional: true + + playwright-core@1.61.0: {} + + playwright@1.61.0: + dependencies: + playwright-core: 1.61.0 + optionalDependencies: + fsevents: 2.3.2 + + typescript@5.9.3: {} + + undici-types@6.21.0: {} diff --git a/docs-tests/pnpm-workspace.yaml b/docs-tests/pnpm-workspace.yaml new file mode 100644 index 0000000000..5b12bd3371 --- /dev/null +++ b/docs-tests/pnpm-workspace.yaml @@ -0,0 +1,5 @@ +# Makes docs-tests its own pnpm workspace root so `pnpm install` here resolves +# this package's deps (Playwright et al.) instead of walking up to the parent +# apify-docs workspace. Keeps the test harness fully isolated from the main +# docs build's dependency tree. +packages: [] diff --git a/docs-tests/tests/auth.fixture.ts b/docs-tests/tests/auth.fixture.ts index b628c9c715..af2824be12 100644 --- a/docs-tests/tests/auth.fixture.ts +++ b/docs-tests/tests/auth.fixture.ts @@ -7,6 +7,21 @@ const BASE_URL = process.env.CONSOLE_STAGING_URL; type StorageState = Awaited>; +// Step logging for the login flow. The worker fixture runs before any test, and +// the `list` reporter prints nothing until a test completes — so without this a +// slow or stuck login is indistinguishable from a crash. Each line carries the +// elapsed time since the step before it, so a frozen step is obvious. Writes to +// stderr to bypass Playwright's per-test stdout buffering and show up live. +function makeLogger() { + let last = Date.now(); + return (step: string): void => { + const now = Date.now(); + const delta = ((now - last) / 1000).toFixed(1); + last = now; + process.stderr.write(`[auth +${delta}s] ${step}\n`); + }; +} + // Worker-scoped authentication. Logs in ONCE per worker with credentials from // the environment (.env locally, GitHub Secrets in CI) and hands the resulting // session to every test as an in-memory storageState. @@ -24,28 +39,46 @@ export const test = base.extend({ ); } + const log = makeLogger(); + log('starting login (worker fixture)'); + const context = await browser.newContext({ baseURL: BASE_URL }); const page = await context.newPage(); - await page.goto('/sign-in'); + log('navigating to /sign-in'); + // The Console is an SPA that never reaches networkidle (it holds live + // connections open), so wait on DOM content and let the form hydrate. + await page.goto('/sign-in', { waitUntil: 'domcontentloaded' }); + log(`loaded ${new URL(page.url()).pathname}`); - // Selectors aren't pinned to the Console DOM; each field tries a few - // resilient locators. If a step misses, the retained trace shows the form. - const email = page - .locator('input[type="email"], input[name="email"], input[name="username"]') - .first(); + // The Apify sign-in is a TWO-STEP native form (verified against staging): + // step 1: #email + a "Next" submit button + // step 2: #password + a "Log in" submit button + // Both steps stay on /sign-in (SPA). The "Continue with Google/GitHub" + // buttons are SSO — never click those; target the native submit buttons. + const email = page.locator('#email, input[type="email"]').first(); + log('waiting for email field'); await email.waitFor({ state: 'visible', timeout: 15_000 }); await email.fill(EMAIL); + log('email filled, clicking Next'); + + await page.getByRole('button', { name: /^next$/i }).click(); - const password = page.locator('input[type="password"], input[name="password"]').first(); + const password = page.locator('#password, input[type="password"]').first(); + log('waiting for password field'); + await password.waitFor({ state: 'visible', timeout: 15_000 }); await password.fill(PASSWORD); + log('password filled, clicking Log in'); - await page.getByRole('button', { name: /sign in|log in|continue/i }).first().click(); + await page.getByRole('button', { name: /^log ?in$/i }).click(); + log('submitted credentials, waiting to leave /sign-in'); // Confirm we left the sign-in page; otherwise creds/selectors are wrong. await page.waitForURL((url) => !/\/sign-in/.test(url.pathname), { timeout: 15_000 }); + log(`logged in, landed on ${new URL(page.url()).pathname}`); const state = await context.storageState(); await context.close(); + log('session captured, handing off to tests'); await use(state); },