diff --git a/docs/commands/doctor.md b/docs/commands/doctor.md index bd650c910..acd006abd 100644 --- a/docs/commands/doctor.md +++ b/docs/commands/doctor.md @@ -86,6 +86,11 @@ Provider readiness validates the selected provider without creating a lease. Blaxel validates the configured API URL, reports whether an API key and workspace are configured, probes the Blaxel API, and lists inventory with `mutation=false`. + Flue validates only local, non-mutating readiness: configured CLI help, + optional version output, configured root/config/env paths, output mode + sanity, `target=node`, and workflow name presence. It reports workflow + discovery as `unchecked` unless Flue exposes a safe read-only discovery + command; it does not run workflows or create sandboxes during doctor. Vercel Sandbox checks the SDK bridge contract, local `sandbox` CLI, read-only `sandbox list --all --limit 1` auth/inventory access, project scoping readiness, and local `vsbx_...` diff --git a/docs/commands/run.md b/docs/commands/run.md index 9adc1459c..57f4712dd 100644 --- a/docs/commands/run.md +++ b/docs/commands/run.md @@ -112,6 +112,14 @@ command through the Blaxel process API, and mirrors the remote exit code. `--keep-on-failure` follow the delegated-run contract; SSH-only run features are rejected. +`--provider flue` is a Node/local delegated-run bridge. Crabbox archives the +checkout, writes a temporary request JSON file, and invokes +`flue run workflow: --target node --input '{"requestFile":"..."}'`. +The configured Flue workflow extracts the archive, runs the command, and prints +a final protocol response JSON. Crabbox v1 rejects non-`node` Flue targets, +persistent lease IDs, retained sessions, SSH-only options, Cloudflare/server +staging, and artifact/download/session promises. + `--provider cloudflare-dynamic-workers` is a module-runtime provider. It accepts Worker module source through `--script` or `--script-stdin`, supports cache and egress controls through `--cloudflare-dynamic-workers-*` flags, and rejects diff --git a/docs/examples/flue/crabbox-runner.mjs b/docs/examples/flue/crabbox-runner.mjs new file mode 100755 index 000000000..44e3b5de8 --- /dev/null +++ b/docs/examples/flue/crabbox-runner.mjs @@ -0,0 +1,245 @@ +#!/usr/bin/env node +import { spawn } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm } from "node:fs/promises"; +import { isAbsolute, join, normalize, relative, resolve } from "node:path"; +import { tmpdir } from "node:os"; + +const PROTOCOL_VERSION = 1; +const OPERATION = "run"; +const DEFAULT_STDOUT_LIMIT = 10 * 1024 * 1024; +const DEFAULT_STDERR_LIMIT = 10 * 1024 * 1024; + +main().catch((error) => { + const message = error instanceof Error ? error.message : String(error); + process.stdout.write( + JSON.stringify({ + protocolVersion: PROTOCOL_VERSION, + operation: OPERATION, + exitCode: 1, + stderr: `${message}\n`, + error: message, + timing: { totalMs: 0, runMs: 0 } + }) + "\n" + ); + process.exitCode = 1; +}); + +async function main() { + const started = Date.now(); + const input = parseFlueInput(process.argv.slice(2)); + const request = validateRequest(JSON.parse(await readFile(input.requestFile, "utf8"))); + const limits = { + stdoutBytes: request.outputLimits?.stdoutBytes || DEFAULT_STDOUT_LIMIT, + stderrBytes: request.outputLimits?.stderrBytes || DEFAULT_STDERR_LIMIT + }; + const stagingRoot = await mkdtemp(join(tmpdir(), "crabbox-flue-runner-")); + try { + await validateArchiveEntries(request.workspaceArchive); + const workspace = workspacePath(stagingRoot, request.workspace); + await mkdir(workspace, { recursive: true }); + await runChecked("tar", ["-xzf", request.workspaceArchive, "-C", workspace], { + cwd: stagingRoot, + env: {}, + limits + }); + const runStarted = Date.now(); + const command = request.command; + const result = await runCommand(command[0], command.slice(1), { + cwd: workspace, + env: commandEnv(request.env || {}), + limits, + timeoutMs: request.timeoutMs || 0 + }); + process.stdout.write( + JSON.stringify({ + protocolVersion: PROTOCOL_VERSION, + operation: OPERATION, + leaseId: request.leaseId, + slug: request.slug, + exitCode: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + timing: { + runMs: Date.now() - runStarted, + totalMs: Date.now() - started + }, + error: result.error + }) + "\n" + ); + process.exitCode = result.exitCode === 0 ? 0 : result.exitCode; + } finally { + await rm(stagingRoot, { recursive: true, force: true }); + } +} + +function parseFlueInput(args) { + const inputIndex = args.indexOf("--input"); + if (inputIndex < 0 || inputIndex + 1 >= args.length) { + throw new Error("missing --input JSON"); + } + const input = JSON.parse(args[inputIndex + 1]); + if (!input || typeof input.requestFile !== "string" || input.requestFile.trim() === "") { + throw new Error("input.requestFile is required"); + } + return input; +} + +function validateRequest(request) { + if (request.protocolVersion !== PROTOCOL_VERSION) { + throw new Error(`unsupported protocolVersion ${request.protocolVersion}`); + } + if (request.operation !== OPERATION) { + throw new Error(`unsupported operation ${request.operation}`); + } + if (request.target !== "node") { + throw new Error(`unsupported target ${request.target}`); + } + if (typeof request.workspaceArchive !== "string" || request.workspaceArchive.trim() === "") { + throw new Error("workspaceArchive is required"); + } + if (typeof request.workspace !== "string" || request.workspace.trim() === "") { + throw new Error("workspace is required"); + } + if (!Array.isArray(request.command) || request.command.length === 0 || typeof request.command[0] !== "string") { + throw new Error("command must be a non-empty argv array"); + } + if (request.timeoutMs && (!Number.isInteger(request.timeoutMs) || request.timeoutMs < 0)) { + throw new Error("timeoutMs must be non-negative"); + } + return request; +} + +async function validateArchiveEntries(archivePath) { + const names = await runChecked("tar", ["-tzf", archivePath], { + cwd: tmpdir(), + env: {}, + limits: { stdoutBytes: DEFAULT_STDOUT_LIMIT, stderrBytes: DEFAULT_STDERR_LIMIT } + }); + if (names.stdoutTruncated) { + throw new Error("archive listing exceeded validation output limit"); + } + for (const rawEntry of names.stdout.split("\n")) { + const entry = rawEntry.trim(); + if (!entry) continue; + const normalized = normalize(entry); + if (isAbsolute(entry) || normalized === ".." || normalized.startsWith("../")) { + throw new Error(`unsafe archive entry ${entry}`); + } + } + + const verbose = await runChecked("tar", ["-tvzf", archivePath], { + cwd: tmpdir(), + env: {}, + limits: { stdoutBytes: DEFAULT_STDOUT_LIMIT, stderrBytes: DEFAULT_STDERR_LIMIT } + }); + if (verbose.stdoutTruncated) { + throw new Error("archive verbose listing exceeded validation output limit"); + } + for (const line of verbose.stdout.split("\n")) { + const type = line.trimStart()[0]; + if (type === "l" || type === "h") { + throw new Error("archive contains link entries"); + } + } +} + +function workspacePath(stagingRoot, requestedWorkspace) { + const suffix = requestedWorkspace.replace(/^[/\\]+/, ""); + const workspace = resolve(stagingRoot, suffix || "workspace"); + if (relative(stagingRoot, workspace).startsWith("..")) { + throw new Error("workspace escaped staging root"); + } + return workspace; +} + +function runChecked(command, args, options) { + return runCommand(command, args, options).then((result) => { + if (result.exitCode !== 0) { + throw new Error(`${command} failed with exit ${result.exitCode}: ${result.stderr || result.stdout}`); + } + return result; + }); +} + +function runCommand(command, args, options) { + return new Promise((resolveRun) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + detached: process.platform !== "win32", + stdio: ["ignore", "pipe", "pipe"] + }); + let stdout = ""; + let stderr = ""; + let stdoutTruncated = false; + let stderrTruncated = false; + let timedOut = false; + const timer = + options.timeoutMs > 0 + ? setTimeout(() => { + timedOut = true; + terminateChild(child, "SIGTERM"); + setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) { + terminateChild(child, "SIGKILL"); + } + }, 5000).unref(); + }, options.timeoutMs) + : null; + child.stdout.on("data", (chunk) => { + const next = appendLimited(stdout, chunk, options.limits.stdoutBytes); + stdout = next.value; + stdoutTruncated = stdoutTruncated || next.truncated; + }); + child.stderr.on("data", (chunk) => { + const next = appendLimited(stderr, chunk, options.limits.stderrBytes); + stderr = next.value; + stderrTruncated = stderrTruncated || next.truncated; + }); + child.on("error", (error) => { + if (timer) clearTimeout(timer); + resolveRun({ exitCode: 1, stdout, stderr, stdoutTruncated, stderrTruncated, error: error.message }); + }); + child.on("close", (code, signal) => { + if (timer) clearTimeout(timer); + const exitCode = timedOut ? 124 : code || (signal ? 1 : 0); + resolveRun({ + exitCode, + stdout, + stderr, + stdoutTruncated, + stderrTruncated, + error: timedOut ? "command timed out" : exitCode === 0 ? "" : `command exited ${exitCode}` + }); + }); + }); +} + +function terminateChild(child, signal) { + if (process.platform !== "win32" && child.pid) { + try { + process.kill(-child.pid, signal); + return; + } catch { + // Fall through to the direct child for platforms or launchers that do + // not preserve the process group. + } + } + child.kill(signal); +} + +function commandEnv(requestEnv) { + const env = {}; + for (const key of ["PATH", "HOME", "TMPDIR", "TEMP", "TMP"]) { + if (process.env[key]) { + env[key] = process.env[key]; + } + } + return { ...env, ...requestEnv }; +} + +function appendLimited(current, chunk, limit) { + const next = current + chunk.toString("utf8"); + if (next.length <= limit) return { value: next, truncated: false }; + return { value: next.slice(0, limit), truncated: true }; +} diff --git a/docs/providers/README.md b/docs/providers/README.md index 501039077..e172ae919 100644 --- a/docs/providers/README.md +++ b/docs/providers/README.md @@ -61,7 +61,7 @@ selection metadata. Regenerate it with `node scripts/generate-provider-matrix.mj `scripts/check-docs.sh` fails when provider registration, metadata, docs paths, or this generated table drift. -Current built-in surface: 67 providers (39 SSH lease, 26 delegated run, 2 service control). +Current built-in surface: 68 providers (39 SSH lease, 27 delegated run, 2 service control). Access terms: @@ -96,6 +96,7 @@ Access terms: | [external](external.md) (`exec-provider`) | built-in; `ssh-lease` · external-provider | Crabbox-managed SSH; `crabbox-sync` · direct only; features: `ssh`, `crabbox-sync`, `cleanup`, `desktop`, `browser`, `code` | `linux`; Configured executable contract | `byo`; GPU: unknown | external executable; contract-defined | Private or organization-specific provider integration | Safety and semantics depend on the configured executable | | [fastapi-cloud](fastapi-cloud.md) (`fastapicloud`, `fastapi`) | specialized; `service-control` · service-control | SSH not applicable; `none` · direct only; features: none | `linux`; FastAPI Cloud app | `cloud`; GPU: unknown | FastAPI Cloud; not exposed | Inspecting FastAPI Cloud app deployment readiness | Cannot execute arbitrary Crabbox run commands or stop apps | | [firecracker](firecracker.md) | built-in; `ssh-lease` · self-hosted-virtualization | Crabbox-managed SSH; `crabbox-sync` · direct only; features: `ssh`, `crabbox-sync`, `cleanup` | `linux`; Firecracker microVM | `self-hosted`; GPU: no | Crabbox direct lifecycle; microVM and local artifact cleanup | Self-hosted Linux KVM host with prepared Firecracker kernel, rootfs, and CNI | Requires Linux, /dev/kvm, Firecracker assets, and a working CNI setup on the host | +| [flue](flue.md) | built-in; `delegated-run` · delegated-sandbox | No SSH; `archive-sync` · direct only; features: `archive-sync` | `linux`; Flue workflow runner | `local`; GPU: unknown | Flue workflow; one-shot workflow exit and temp-file cleanup | Local Flue workflow delegation with Crabbox archive sync | Node target only in v1; no SSH, coordinator, retained sessions, or Cloudflare/server staging | | [freestyle](freestyle.md) | built-in; `delegated-run` · delegated-sandbox | No SSH; `archive-sync` · direct only; features: `archive-sync`, `run-session` | `linux`; Freestyle VM | `provider-managed`; GPU: unknown | Freestyle; provider VM cleanup | Hosted delegated Linux VM execution | No Crabbox-managed SSH path | | [gcp](gcp.md) (`google`, `google-cloud`) | built-in; `ssh-lease` · brokerable-cloud | Crabbox-managed SSH; `crabbox-sync` · coordinator optional; features: `ssh`, `crabbox-sync`, `cleanup`, `tailscale` | `linux`; Google Compute Engine VM | `cloud`; GPU: optional | Crabbox or coordinator; instance and firewall cleanup | Linux compute with broad machine selection | Project, IAM, quota, and firewall setup required | | [hetzner](hetzner.md) | built-in; `ssh-lease` · brokerable-cloud | Crabbox-managed SSH; `crabbox-sync` · coordinator optional; features: `ssh`, `crabbox-sync`, `cleanup`, `desktop`, `browser`, `code`, `tailscale` | `linux`; Hetzner Cloud server | `cloud`; GPU: no | Crabbox or coordinator; server delete | Cost-effective high-CPU Linux VM | Linux-only and capacity varies by location | diff --git a/docs/providers/flue.md b/docs/providers/flue.md new file mode 100644 index 000000000..ca7e44ab3 --- /dev/null +++ b/docs/providers/flue.md @@ -0,0 +1,252 @@ +# Flue Provider + +Read when: + +- choosing `provider: flue`; +- running a Crabbox checkout through a Flue workflow; +- changing `internal/providers/flue`. + +Flue is a delegated-run provider. Crabbox builds a local archive of the current +Git checkout, writes a versioned request JSON file, and asks the local `flue` +CLI to run a named workflow. Flue owns the sandbox, command transport, and +workflow implementation. Crabbox owns provider selection, archive creation, +request-file cleanup, output parsing, timing summaries, and doctor/readiness +reporting. + +This is not an SSH lease. It does not go through the Crabbox coordinator, does +not expose ports, does not create a persistent Crabbox session, and does not +support desktop, browser, code-server, Tailscale, retained leases, or remote +artifact promises in v1. + +## When To Use + +Use `flue` when you already have a local Flue project with a workflow that can +consume Crabbox's request-file protocol and run a one-shot Linux command: + +```sh +crabbox doctor --provider flue --flue-root ./flue-runner +crabbox run --provider flue --flue-root ./flue-runner --flue-workflow crabbox-runner -- echo ok +crabbox run --provider flue --flue-root ./flue-runner --flue-workdir /workspace/my-app -- pnpm test +``` + +Use an SSH-lease provider such as `aws`, `hetzner`, `local-container`, or +`ssh` when you need a persistent machine, normal SSH access, Crabbox-managed +rsync, ports, copies, desktop/browser/code-server, or coordinator scheduling. + +## Prerequisites + +- Install Flue so `flue` is on `PATH`, or configure `--flue-cli` / + `flue.cliPath`. +- Create a local Flue project root and pass it with `--flue-root`, or run + Crabbox from a directory where the Flue CLI can discover the project. +- Add a workflow named `crabbox-runner`, or pass another workflow name with + `--flue-workflow`. +- Keep the Flue target as `node`. Crabbox v1 passes host-local archive and + request-file paths, so Cloudflare and remote server targets need a future + upload or HTTP staging contract before they can work. +- Keep provider secrets in Flue/project/provider environment mechanisms such as + `--flue-env`, not in Crabbox argv. + +## Commands + +```sh +crabbox doctor --provider flue --flue-root ./flue-runner +crabbox doctor --provider flue --flue-cli /opt/flue/bin/flue --flue-root ./flue-runner --json +crabbox run --provider flue --flue-root ./flue-runner --flue-workflow crabbox-runner -- echo ok +crabbox run --provider flue --flue-root ./flue-runner --flue-config flue.config.ts --flue-env .env -- pnpm test +``` + +Crabbox invokes Flue as: + +```text +flue run workflow: --target node --input '{"requestFile":""}' [--root ] [--config ] [--env ] [--output ] +``` + +The `--input` payload intentionally contains only a pointer to a temporary +request file. The full command, environment, archive path, timeout, and output +limits stay in that request file so secrets are not copied into the Flue argv. +Crabbox removes both the request file and the archive after the Flue process +returns. + +## Config + +```yaml +provider: flue +target: linux +flue: + cliPath: flue + root: ./flue-runner + workflow: crabbox-runner + target: node + config: flue.config.ts + envFile: .env + output: json + workdir: /workspace/crabbox + timeoutSecs: 1800 +``` + +Provider flags: + +```text +--flue-cli +--flue-root +--flue-workflow +--flue-target node +--flue-config +--flue-env +--flue-output +--flue-workdir +--flue-timeout-secs +``` + +Environment overrides: + +```text +CRABBOX_FLUE_CLI +CRABBOX_FLUE_ROOT +CRABBOX_FLUE_WORKFLOW +CRABBOX_FLUE_TARGET +CRABBOX_FLUE_CONFIG +CRABBOX_FLUE_ENV +CRABBOX_FLUE_OUTPUT +CRABBOX_FLUE_WORKDIR +CRABBOX_FLUE_TIMEOUT_SECS +``` + +Precedence follows the normal Crabbox order: + +```text +flags > env > repo config > user config > defaults +``` + +## Request Protocol + +Flue receives a small CLI input object: + +```json +{"requestFile":"/tmp/crabbox-flue-request-123.json"} +``` + +The request file is `0600` and contains: + +```json +{ + "protocolVersion": 1, + "operation": "run", + "workflow": "crabbox-runner", + "target": "node", + "workspaceArchive": "/tmp/crabbox-flue-sync-123.tgz", + "workspace": "/workspace/crabbox", + "command": ["echo", "ok"], + "env": {}, + "timeoutMs": 1800000, + "outputLimits": { + "stdoutBytes": 10485760, + "stderrBytes": 10485760 + } +} +``` + +The workflow must print a final JSON response on stdout. It may print progress +before that, but the final parseable JSON object should be last: + +```json +{ + "protocolVersion": 1, + "operation": "run", + "exitCode": 0, + "stdout": "ok\n", + "stderr": "", + "timing": { + "runMs": 42, + "totalMs": 150 + } +} +``` + +`exitCode` is the delegated command's exit code. `stdout` and `stderr` are +replayed by Crabbox onto the local Crabbox stdout/stderr streams. A non-zero +`exitCode` makes `crabbox run` exit non-zero with the same command failure +classification. + +## Runner Example + +The example runner at +[`docs/examples/flue/crabbox-runner.mjs`](../examples/flue/crabbox-runner.mjs) +is intentionally dependency-free Node code. It validates `protocolVersion`, +reads the request file, checks the archive member names before extraction, +stages the archive, runs the command, enforces the request timeout, returns +structured stdout/stderr/exit/timing, and avoids logging request contents or +environment values. + +Use it as the command body inside a Flue workflow, or as the implementation +model for a TypeScript workflow that uses Flue's sandbox APIs: + +```sh +node docs/examples/flue/crabbox-runner.mjs --input '{"requestFile":"/tmp/crabbox-flue-request.json"}' +``` + +The example is executable as a local protocol fixture. It is not a generated +Flue project and does not configure Cloudflare, server mode, retained sessions, +live streaming, or artifact upload/download. + +## Doctor + +`crabbox doctor --provider flue` is non-mutating. It does not run the configured +workflow and never creates a sandbox. The provider doctor checks: + +- `flue --help` through the configured CLI path. +- `flue --version` as optional, non-authoritative context. +- configured `flue.root`, `flue.config`, and `flue.envFile` path readability. +- `flue.output` string sanity. +- `flue.target=node`. +- configured workflow name presence, while reporting workflow discovery as + `unchecked` unless Flue exposes a safe read-only discovery command. + +Example JSON check fields include `mutation=false`, `target=node`, and +`discoverability=unchecked`. A missing CLI, unreadable configured root, missing +configured config/env file, or non-`node` target fails doctor. + +## Lifecycle And Capabilities + +- Kind: delegated-run. +- Family: `flue`. +- Canonical provider: `flue`. +- Targets: Linux. +- Coordinator: never. Crabbox shells out to the local Flue CLI. +- Sync: archive-sync. Crabbox creates a tar archive and Flue extracts it. +- SSH: unsupported. +- Persistent lifecycle: unsupported. `list` returns no leases; `warmup`, + `status`, and `stop` report one-shot unsupported errors. +- Desktop, browser, code-server, VNC, Tailscale, ports, copies, checkpoints, + retained sessions, live streaming, run artifacts, and run downloads: + unsupported in v1. + +Unsupported generic sizing flags such as `--class` and `--type` are rejected. +`--no-sync`, `--sync-only`, `--keep`, `--keep-on-failure`, persistent `--id`, +desktop/browser/code-server, Tailscale, local patch application, scripts, +fresh-PR hydration, proof emission, and SSH-only capture options are rejected +unless a future provider version advertises a matching capability. + +## Smoke + +Build Crabbox and run the deterministic CLI surfaces: + +```sh +go build -trimpath -o bin/crabbox ./cmd/crabbox +(cd /tmp && /path/to/crabbox/bin/crabbox providers --json) +(cd /tmp && /path/to/crabbox/bin/crabbox doctor --provider flue --flue-root /path/to/flue-runner) +``` + +For a fake or fixture Flue CLI, make the fake `flue` command read the +`--input` pointer, invoke the runner example, and return the final response JSON: + +```sh +crabbox run --provider flue --flue-cli /path/to/fake-flue --flue-root /path/to/fixture --flue-workflow crabbox-runner -- echo ok +``` + +If a real local Flue install or workflow prerequisites are unavailable, classify +that live proof as `environment_blocked`. Keep the fake CLI and Go protocol +tests as deterministic proof that Crabbox builds the request-file bridge, +cleans up temporary files, parses the response, and propagates command output +and exit status. diff --git a/docs/providers/provider-metadata.json b/docs/providers/provider-metadata.json index d8f001dbe..84984ef7b 100644 --- a/docs/providers/provider-metadata.json +++ b/docs/providers/provider-metadata.json @@ -349,6 +349,20 @@ "caveat": "Requires Linux, /dev/kvm, Firecracker assets, and a working CNI setup on the host", "docs": "firecracker.md" }, + "flue": { + "status": "built-in", + "category": "delegated-sandbox", + "substrate": "Flue workflow runner", + "location": "local", + "ssh": "no", + "sync": "archive-sync", + "gpu": "unknown", + "lifecycle": "Flue workflow", + "cleanup": "one-shot workflow exit and temp-file cleanup", + "bestFit": "Local Flue workflow delegation with Crabbox archive sync", + "caveat": "Node target only in v1; no SSH, coordinator, retained sessions, or Cloudflare/server staging", + "docs": "flue.md" + }, "freestyle": { "status": "built-in", "category": "delegated-sandbox", diff --git a/internal/cli/config.go b/internal/cli/config.go index 888dd3ca5..cec61f846 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -177,6 +177,7 @@ type Config struct { Freestyle FreestyleConfig Tenki TenkiConfig Tensorlake TensorlakeConfig + Flue FlueConfig OpenComputer OpenComputerConfig CodeSandbox CodeSandboxConfig OpenSandbox OpenSandboxConfig @@ -697,6 +698,21 @@ type TensorlakeConfig struct { NoInternet bool } +// FlueConfig configures the delegated Flue provider. It is intentionally +// non-secret: credentials stay in Flue-managed env/config files, not Crabbox +// config or argv. +type FlueConfig struct { + CLIPath string + Root string + Workflow string + Target string + Config string + EnvFile string + Output string + Workdir string + TimeoutSecs int +} + // OpenComputerConfig configures the delegated OpenComputer provider, which // talks to the OpenComputer REST API. The API key is intentionally absent: it // is read at runtime from CRABBOX_OPENCOMPUTER_API_KEY / OPENCOMPUTER_API_KEY @@ -2590,6 +2606,13 @@ func baseConfig() Config { MemoryMB: 1024, DiskMB: 10240, }, + Flue: FlueConfig{ + CLIPath: "flue", + Workflow: "crabbox-runner", + Target: "node", + Workdir: "/workspace/crabbox", + TimeoutSecs: 1800, + }, OpenComputer: OpenComputerConfig{ // APIURL is intentionally unset here so the `oc` config file's // api_url is honored before the built-in default; the provider @@ -2859,6 +2882,7 @@ type fileConfig struct { Freestyle *fileFreestyleConfig `yaml:"freestyle,omitempty"` Tenki *fileTenkiConfig `yaml:"tenki,omitempty"` Tensorlake *fileTensorlakeConfig `yaml:"tensorlake,omitempty"` + Flue *fileFlueConfig `yaml:"flue,omitempty"` OpenComputer *fileOpenComputerConfig `yaml:"openComputer,omitempty"` CodeSandbox *fileCodeSandboxConfig `yaml:"codeSandbox,omitempty"` OpenSandbox *fileOpenSandboxConfig `yaml:"openSandbox,omitempty"` @@ -3491,6 +3515,18 @@ type fileTensorlakeConfig struct { NoInternet *bool `yaml:"noInternet,omitempty"` } +type fileFlueConfig struct { + CLIPath string `yaml:"cliPath,omitempty"` + Root string `yaml:"root,omitempty"` + Workflow string `yaml:"workflow,omitempty"` + Target string `yaml:"target,omitempty"` + Config string `yaml:"config,omitempty"` + EnvFile string `yaml:"envFile,omitempty"` + Output string `yaml:"output,omitempty"` + Workdir string `yaml:"workdir,omitempty"` + TimeoutSecs *int `yaml:"timeoutSecs,omitempty"` +} + type fileOpenComputerConfig struct { Workdir string `yaml:"workdir,omitempty"` CPU *int `yaml:"cpu,omitempty"` @@ -5683,6 +5719,38 @@ func applyFileConfigWithTrust(cfg *Config, file fileConfig, trusted bool) error cfg.Tensorlake.NoInternet = *file.Tensorlake.NoInternet } } + if file.Flue != nil { + if file.Flue.CLIPath != "" { + cfg.Flue.CLIPath = file.Flue.CLIPath + } + if file.Flue.Root != "" { + cfg.Flue.Root = expandUserPath(file.Flue.Root) + } + if file.Flue.Workflow != "" { + cfg.Flue.Workflow = file.Flue.Workflow + } + if file.Flue.Target != "" { + cfg.Flue.Target = file.Flue.Target + } + if file.Flue.Config != "" { + cfg.Flue.Config = expandUserPath(file.Flue.Config) + } + if file.Flue.EnvFile != "" { + cfg.Flue.EnvFile = expandUserPath(file.Flue.EnvFile) + } + if file.Flue.Output != "" { + cfg.Flue.Output = file.Flue.Output + } + if file.Flue.Workdir != "" { + cfg.Flue.Workdir = file.Flue.Workdir + } + if file.Flue.TimeoutSecs != nil { + if *file.Flue.TimeoutSecs < 0 { + return exit(2, "flue timeoutSecs must be non-negative") + } + cfg.Flue.TimeoutSecs = *file.Flue.TimeoutSecs + } + } if file.OpenComputer != nil { if file.OpenComputer.Workdir != "" { cfg.OpenComputer.Workdir = file.OpenComputer.Workdir @@ -7508,6 +7576,24 @@ func applyEnv(cfg *Config) error { cfg.OpenComputer.Burst = v } var err error + cfg.Flue.CLIPath = getenv("CRABBOX_FLUE_CLI", cfg.Flue.CLIPath) + if root := os.Getenv("CRABBOX_FLUE_ROOT"); root != "" { + cfg.Flue.Root = expandUserPath(root) + } + cfg.Flue.Workflow = getenv("CRABBOX_FLUE_WORKFLOW", cfg.Flue.Workflow) + cfg.Flue.Target = getenv("CRABBOX_FLUE_TARGET", cfg.Flue.Target) + if config := os.Getenv("CRABBOX_FLUE_CONFIG"); config != "" { + cfg.Flue.Config = expandUserPath(config) + } + if envFile := os.Getenv("CRABBOX_FLUE_ENV"); envFile != "" { + cfg.Flue.EnvFile = expandUserPath(envFile) + } + cfg.Flue.Output = getenv("CRABBOX_FLUE_OUTPUT", cfg.Flue.Output) + cfg.Flue.Workdir = getenv("CRABBOX_FLUE_WORKDIR", cfg.Flue.Workdir) + cfg.Flue.TimeoutSecs, err = getenvNonNegativeInt("CRABBOX_FLUE_TIMEOUT_SECS", cfg.Flue.TimeoutSecs) + if err != nil { + return err + } cfg.CodeSandbox.TemplateID = getenv("CRABBOX_CODESANDBOX_TEMPLATE_ID", cfg.CodeSandbox.TemplateID) cfg.CodeSandbox.Workdir = getenv("CRABBOX_CODESANDBOX_WORKDIR", cfg.CodeSandbox.Workdir) cfg.CodeSandbox.VMTier = getenv("CRABBOX_CODESANDBOX_VM_TIER", cfg.CodeSandbox.VMTier) diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index 6ba65021a..da468e6ce 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -285,6 +285,15 @@ func clearConfigEnv(t *testing.T) { "CRABBOX_TENSORLAKE_DISK_MB", "CRABBOX_TENSORLAKE_TIMEOUT_SECS", "CRABBOX_TENSORLAKE_NO_INTERNET", + "CRABBOX_FLUE_CLI", + "CRABBOX_FLUE_ROOT", + "CRABBOX_FLUE_WORKFLOW", + "CRABBOX_FLUE_TARGET", + "CRABBOX_FLUE_CONFIG", + "CRABBOX_FLUE_ENV", + "CRABBOX_FLUE_OUTPUT", + "CRABBOX_FLUE_WORKDIR", + "CRABBOX_FLUE_TIMEOUT_SECS", "CRABBOX_DOCKER_SANDBOX_CLI", "CRABBOX_DOCKER_SANDBOX_AGENT", "CRABBOX_DOCKER_SANDBOX_TEMPLATE", @@ -4607,6 +4616,16 @@ tensorlake: diskMB: 30000 timeoutSecs: 1800 noInternet: true +flue: + cliPath: /usr/local/bin/flue + root: ~/src/flue-project + workflow: workflow:crabbox + target: node + config: ~/.config/flue/flue.config.ts + envFile: ~/.config/flue/.env + output: json + workdir: /workspace/flue-test + timeoutSecs: 1200 openComputer: apiUrl: https://opencomputer.example.test workdir: /workspace/oc-test @@ -4825,6 +4844,9 @@ ssh: if cfg.Tensorlake.APIURL != "https://api.tensorlake.example.test" || cfg.Tensorlake.CLIPath != "/usr/local/bin/tl" || cfg.Tensorlake.Image != "ubuntu-22.04" || cfg.Tensorlake.Snapshot != "snap-tl" || cfg.Tensorlake.OrganizationID != "org-tl" || cfg.Tensorlake.ProjectID != "proj-tl" || cfg.Tensorlake.Namespace != "ns-tl" || cfg.Tensorlake.Workdir != "/workspace/crabbox-test" || cfg.Tensorlake.CPUs != 4 || cfg.Tensorlake.MemoryMB != 8192 || cfg.Tensorlake.DiskMB != 30000 || cfg.Tensorlake.TimeoutSecs != 1800 || !cfg.Tensorlake.NoInternet { t.Fatalf("tensorlake config not loaded: %#v", cfg.Tensorlake) } + if cfg.Flue.CLIPath != "/usr/local/bin/flue" || cfg.Flue.Root != filepath.Join(home, "src", "flue-project") || cfg.Flue.Workflow != "workflow:crabbox" || cfg.Flue.Target != "node" || cfg.Flue.Config != filepath.Join(home, ".config", "flue", "flue.config.ts") || cfg.Flue.EnvFile != filepath.Join(home, ".config", "flue", ".env") || cfg.Flue.Output != "json" || cfg.Flue.Workdir != "/workspace/flue-test" || cfg.Flue.TimeoutSecs != 1200 { + t.Fatalf("flue config not loaded: %#v", cfg.Flue) + } if cfg.OpenComputer.APIURL != "" || cfg.OpenComputer.Workdir != "/workspace/oc-test" || cfg.OpenComputer.CPU != 8 || cfg.OpenComputer.MemoryMB != 16384 || cfg.OpenComputer.TimeoutSecs != 600 || cfg.OpenComputer.ExecTimeoutSecs != 7200 { t.Fatalf("opencomputer config not loaded: %#v", cfg.OpenComputer) } @@ -5195,6 +5217,15 @@ func TestEnvOverridesConfig(t *testing.T) { t.Setenv("CRABBOX_TENSORLAKE_DISK_MB", "20480") t.Setenv("CRABBOX_TENSORLAKE_TIMEOUT_SECS", "900") t.Setenv("CRABBOX_TENSORLAKE_NO_INTERNET", "true") + t.Setenv("CRABBOX_FLUE_CLI", "/opt/flue/bin/flue") + t.Setenv("CRABBOX_FLUE_ROOT", "~/flue-env") + t.Setenv("CRABBOX_FLUE_WORKFLOW", "env-runner") + t.Setenv("CRABBOX_FLUE_TARGET", "node") + t.Setenv("CRABBOX_FLUE_CONFIG", "~/flue.config.ts") + t.Setenv("CRABBOX_FLUE_ENV", "~/flue.env") + t.Setenv("CRABBOX_FLUE_OUTPUT", "json") + t.Setenv("CRABBOX_FLUE_WORKDIR", "/workspace/flue-env") + t.Setenv("CRABBOX_FLUE_TIMEOUT_SECS", "777") t.Setenv("OPENCOMPUTER_API_URL", "https://oc-file.example") t.Setenv("CRABBOX_OPENCOMPUTER_API_URL", "https://oc-env.example") t.Setenv("CRABBOX_OPENCOMPUTER_WORKDIR", "/workspace/oc-env") @@ -5413,6 +5444,9 @@ func TestEnvOverridesConfig(t *testing.T) { if cfg.Tensorlake.APIKey != "tl-api-env" || cfg.Tensorlake.APIURL != "https://api.tl-env.example" || cfg.Tensorlake.CLIPath != "/opt/tl/bin/tensorlake" || cfg.Tensorlake.Image != "ubuntu:tl-env" || cfg.Tensorlake.Snapshot != "snap-tl-env" || cfg.Tensorlake.OrganizationID != "org-tl-env" || cfg.Tensorlake.ProjectID != "proj-tl-env" || cfg.Tensorlake.Namespace != "ns-tl-env" || cfg.Tensorlake.Workdir != "/workspace/tl-env" || cfg.Tensorlake.CPUs != 2.5 || cfg.Tensorlake.MemoryMB != 4096 || cfg.Tensorlake.DiskMB != 20480 || cfg.Tensorlake.TimeoutSecs != 900 || !cfg.Tensorlake.NoInternet { t.Fatalf("unexpected tensorlake env: %#v", cfg.Tensorlake) } + if cfg.Flue.CLIPath != "/opt/flue/bin/flue" || cfg.Flue.Root != filepath.Join(home, "flue-env") || cfg.Flue.Workflow != "env-runner" || cfg.Flue.Target != "node" || cfg.Flue.Config != filepath.Join(home, "flue.config.ts") || cfg.Flue.EnvFile != filepath.Join(home, "flue.env") || cfg.Flue.Output != "json" || cfg.Flue.Workdir != "/workspace/flue-env" || cfg.Flue.TimeoutSecs != 777 { + t.Fatalf("unexpected flue env: %#v", cfg.Flue) + } if cfg.OpenComputer.APIURL != "https://oc-env.example" || cfg.OpenComputer.Workdir != "/workspace/oc-env" || cfg.OpenComputer.CPU != 6 || cfg.OpenComputer.MemoryMB != 12288 || cfg.OpenComputer.TimeoutSecs != 1200 || cfg.OpenComputer.ExecTimeoutSecs != 2400 { t.Fatalf("unexpected opencomputer env: %#v", cfg.OpenComputer) } @@ -5485,6 +5519,15 @@ func TestApplyEnvRejectsNegativeOpenSandboxTimeouts(t *testing.T) { } } +func TestApplyEnvRejectsNegativeFlueTimeout(t *testing.T) { + t.Setenv("CRABBOX_FLUE_TIMEOUT_SECS", "-1") + cfg := baseConfig() + err := applyEnv(&cfg) + if err == nil || !strings.Contains(err.Error(), "CRABBOX_FLUE_TIMEOUT_SECS must be non-negative") { + t.Fatalf("err=%v, want negative timeout rejection", err) + } +} + func TestExplicitProviderImagesSurvivePortableOSDefaults(t *testing.T) { clearConfigEnv(t) home := t.TempDir() diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index f94507eaa..c11f202aa 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -351,7 +351,13 @@ func (a App) doctor(ctx context.Context, args []string) error { doctorProvider, doctorSupported := providerDef.(DoctorProvider) if doctorSupported { - if err := validateProviderConfig(cfg); err != nil { + var err error + if validator, ok := providerDef.(DoctorConfigValidator); ok { + err = validator.ValidateDoctorConfig(cfg) + } else { + err = validateProviderConfig(cfg) + } + if err != nil { class := doctorErrorClass(err) hint := doctorErrorHint(providerDef.Name(), class) record("failed", "provider", fmt.Sprintf("provider=%s class=%s hint=%s %v", providerDef.Name(), class, hint, err), map[string]string{"provider": providerDef.Name(), "class": class, "hint": hint, "error": err.Error()}) diff --git a/internal/cli/provider_backend.go b/internal/cli/provider_backend.go index 55ea75318..1653fd56f 100644 --- a/internal/cli/provider_backend.go +++ b/internal/cli/provider_backend.go @@ -78,6 +78,10 @@ type DoctorProvider interface { ConfigureDoctor(cfg Config, rt Runtime) (DoctorBackend, error) } +type DoctorConfigValidator interface { + ValidateDoctorConfig(cfg Config) error +} + type DoctorBackend interface { Backend Doctor(ctx context.Context, req DoctorRequest) (DoctorResult, error) diff --git a/internal/cli/provider_categories_generated.go b/internal/cli/provider_categories_generated.go index ffa9baad1..2cd697bb8 100644 --- a/internal/cli/provider_categories_generated.go +++ b/internal/cli/provider_categories_generated.go @@ -28,6 +28,7 @@ var benchmarkProviderCategories = map[string]string{ "external": "external-provider", "fastapi-cloud": "service-control", "firecracker": "self-hosted-virtualization", + "flue": "delegated-sandbox", "freestyle": "delegated-sandbox", "gcp": "brokerable-cloud", "hetzner": "brokerable-cloud", diff --git a/internal/providers/all/all.go b/internal/providers/all/all.go index 951183bff..d10744cfa 100644 --- a/internal/providers/all/all.go +++ b/internal/providers/all/all.go @@ -26,6 +26,7 @@ import ( _ "github.com/openclaw/crabbox/internal/providers/external" _ "github.com/openclaw/crabbox/internal/providers/fastapicloud" _ "github.com/openclaw/crabbox/internal/providers/firecracker" + _ "github.com/openclaw/crabbox/internal/providers/flue" _ "github.com/openclaw/crabbox/internal/providers/freestyle" _ "github.com/openclaw/crabbox/internal/providers/gcp" _ "github.com/openclaw/crabbox/internal/providers/hetzner" diff --git a/internal/providers/all/all_test.go b/internal/providers/all/all_test.go index 467abee5c..aa6317b39 100644 --- a/internal/providers/all/all_test.go +++ b/internal/providers/all/all_test.go @@ -267,6 +267,34 @@ func TestBlaxelRegistersCanonicalWithoutAliases(t *testing.T) { } } +func TestFlueRegistersAsDelegatedRunWithoutAliases(t *testing.T) { + provider, err := core.ProviderFor("flue") + if err != nil { + t.Fatalf("ProviderFor(flue): %v", err) + } + if provider.Name() != "flue" { + t.Fatalf("ProviderFor(flue).Name=%q", provider.Name()) + } + if provider.Aliases() != nil { + t.Fatalf("flue aliases=%v, want none", provider.Aliases()) + } + spec := provider.Spec() + if spec.Family != "flue" || spec.Kind != core.ProviderKindDelegatedRun || spec.Coordinator != core.CoordinatorNever { + t.Fatalf("flue spec=%#v", spec) + } + if len(spec.Targets) != 1 || spec.Targets[0].OS != core.TargetLinux { + t.Fatalf("flue targets=%#v", spec.Targets) + } + if !spec.Features.Has(core.FeatureArchiveSync) || len(spec.Features) != 1 { + t.Fatalf("flue features=%v, want archive-sync only", spec.Features) + } + for _, alias := range []string{"flue-runner", "flue-provider", "fl"} { + if got, err := core.ProviderFor(alias); err == nil && got.Name() == "flue" { + t.Fatalf("%q alias unexpectedly resolves to flue", alias) + } + } +} + func TestAnthropicSandboxRuntimeRegistersCanonicalAndAlias(t *testing.T) { for _, name := range []string{"anthropic-sandbox-runtime", "srt"} { provider, err := core.ProviderFor(name) @@ -1148,6 +1176,7 @@ func allBuiltInProviderNames() []string { "external", "fastapi-cloud", "firecracker", + "flue", "freestyle", "gcp", "hetzner", diff --git a/internal/providers/flue/backend_test.go b/internal/providers/flue/backend_test.go new file mode 100644 index 000000000..532d233ea --- /dev/null +++ b/internal/providers/flue/backend_test.go @@ -0,0 +1,397 @@ +package flue + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + + core "github.com/openclaw/crabbox/internal/cli" +) + +func TestRunBuildsArchiveRequestParsesNoisyResponseAndCleansUp(t *testing.T) { + repo := newGitRepo(t) + secret := "secret-token-value" + var requestPath, archivePath string + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + input := decodeCLIInputArg(t, req.Args) + requestPath = input.RequestFile + if strings.Contains(strings.Join(req.Args, " "), secret) { + t.Fatalf("secret leaked in flue argv: %v", req.Args) + } + data, err := os.ReadFile(requestPath) + if err != nil { + t.Fatalf("read request file during spawn: %v", err) + } + protocolReq, err := ParseRequest(data) + if err != nil { + t.Fatalf("parse request: %v", err) + } + archivePath = protocolReq.WorkspaceArchive + if _, err := os.Stat(archivePath); err != nil { + t.Fatalf("archive not present during spawn: %v", err) + } + if protocolReq.Workspace != "/workspace/test" || !reflect.DeepEqual(protocolReq.Command, []string{"echo", "ok"}) { + t.Fatalf("protocol request=%#v", protocolReq) + } + if protocolReq.Env["SECRET_TOKEN"] != secret { + t.Fatalf("request env=%#v", protocolReq.Env) + } + return LocalCommandResult{ExitCode: 0, Stdout: "progress line\n" + mustResponseJSON(t, Response{ + ProtocolVersion: protocolVersion, + Operation: operationRun, + LeaseID: protocolReq.LeaseID, + Slug: protocolReq.Slug, + ExitCode: 0, + Stdout: "ok\n", + Stderr: "warn\n", + Timing: ResponseTiming{RunMs: 42, TotalMs: 50}, + }) + "\n"}, nil + }} + cfg := testConfig() + cfg.Flue.Workdir = "/workspace/test" + var stdout, stderr bytes.Buffer + backend := testBackend(cfg, runner, &stdout, &stderr) + result, err := backend.Run(context.Background(), RunRequest{ + Repo: Repo{Name: "my-app", Root: repo}, + Command: []string{"echo", "ok"}, + Env: map[string]string{"SECRET_TOKEN": secret}, + EnvSummary: true, + TimingJSON: true, + }) + if err != nil { + t.Fatalf("Run err=%v", err) + } + if result.Provider != providerName || !result.SyncDelegated || result.ExitCode != 0 || result.Command.Milliseconds() != 42 { + t.Fatalf("result=%#v", result) + } + if stdout.String() != "ok\n" || !strings.Contains(stderr.String(), "warn\n") || !strings.Contains(stderr.String(), `"syncDelegated":true`) { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + call := runner.onlyCall(t) + wantPrefix := []string{"run", "workflow:crabbox-runner", "--target", "node", "--input"} + if len(call.Args) < len(wantPrefix) || !reflect.DeepEqual(call.Args[:len(wantPrefix)], wantPrefix) { + t.Fatalf("args=%#v want prefix %#v", call.Args, wantPrefix) + } + if _, err := os.Stat(requestPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("request file cleanup stat err=%v", err) + } + if _, err := os.Stat(archivePath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("archive cleanup stat err=%v", err) + } +} + +func TestRunReturnsCommandExitCodeFromProtocolResponse(t *testing.T) { + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + input := decodeCLIInputArg(t, req.Args) + protocolReq := readProtocolRequest(t, input.RequestFile) + return LocalCommandResult{ExitCode: 0, Stdout: mustResponseJSON(t, Response{ + ProtocolVersion: protocolVersion, + Operation: operationRun, + LeaseID: protocolReq.LeaseID, + Slug: protocolReq.Slug, + ExitCode: 7, + Stdout: "partial\n", + Stderr: "failed\n", + })}, nil + }} + var stdout, stderr bytes.Buffer + result, err := testBackend(testConfig(), runner, &stdout, &stderr).Run(context.Background(), RunRequest{ + Repo: Repo{Name: "my-app", Root: newGitRepo(t)}, + Command: []string{"false"}, + }) + var exitErr core.ExitError + if !core.AsExitError(err, &exitErr) || exitErr.Code != 7 { + t.Fatalf("Run err=%v result=%#v", err, result) + } + if result.ExitCode != 7 || result.Status != core.RunStatusFailed || result.ErrorKind != core.RunErrorCommandExit { + t.Fatalf("result=%#v", result) + } + if stdout.String() != "partial\n" || !strings.Contains(stderr.String(), "failed\n") { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestRunParsesCommandFailureResponseFromNonZeroFlueProcess(t *testing.T) { + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + input := decodeCLIInputArg(t, req.Args) + protocolReq := readProtocolRequest(t, input.RequestFile) + return LocalCommandResult{ExitCode: 7, Stdout: mustResponseJSON(t, Response{ + ProtocolVersion: protocolVersion, + Operation: operationRun, + LeaseID: protocolReq.LeaseID, + Slug: protocolReq.Slug, + ExitCode: 7, + Stdout: "partial\n", + Stderr: "failed\n", + Error: "command exited 7", + })}, errors.New("exit status 7") + }} + var stdout, stderr bytes.Buffer + result, err := testBackend(testConfig(), runner, &stdout, &stderr).Run(context.Background(), RunRequest{ + Repo: Repo{Name: "my-app", Root: newGitRepo(t)}, + Command: []string{"false"}, + }) + var exitErr core.ExitError + if !core.AsExitError(err, &exitErr) || exitErr.Code != 7 { + t.Fatalf("Run err=%v result=%#v", err, result) + } + if result.ExitCode != 7 || result.Status != core.RunStatusFailed || result.ErrorKind != core.RunErrorCommandExit { + t.Fatalf("result=%#v", result) + } + if stdout.String() != "partial\n" || !strings.Contains(stderr.String(), "failed\n") || strings.Contains(err.Error(), "workflow failed") { + t.Fatalf("stdout=%q stderr=%q err=%v", stdout.String(), stderr.String(), err) + } +} + +func TestRunKeepsWorkflowFailureWhenNonZeroFlueProcessReportsSuccess(t *testing.T) { + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + input := decodeCLIInputArg(t, req.Args) + protocolReq := readProtocolRequest(t, input.RequestFile) + return LocalCommandResult{ExitCode: 2, Stdout: mustResponseJSON(t, Response{ + ProtocolVersion: protocolVersion, + Operation: operationRun, + LeaseID: protocolReq.LeaseID, + Slug: protocolReq.Slug, + ExitCode: 0, + Stdout: "ok\n", + }), Stderr: "flue transport failed"}, errors.New("exit status 2") + }} + var stdout, stderr bytes.Buffer + result, err := testBackend(testConfig(), runner, &stdout, &stderr).Run(context.Background(), RunRequest{ + Repo: Repo{Name: "my-app", Root: newGitRepo(t)}, + Command: []string{"true"}, + }) + var exitErr core.ExitError + if !core.AsExitError(err, &exitErr) || exitErr.Code != 2 { + t.Fatalf("Run err=%v result=%#v", err, result) + } + if result.ExitCode != 2 || result.Status != core.RunStatusFailed { + t.Fatalf("result=%#v", result) + } + if stdout.String() != "" || !strings.Contains(err.Error(), "flue workflow failed") { + t.Fatalf("stdout=%q stderr=%q err=%v", stdout.String(), stderr.String(), err) + } +} + +func TestRunRejectsMalformedResponseWithoutLeakingEnv(t *testing.T) { + secret := "very-secret-value" + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + return LocalCommandResult{ExitCode: 0, Stdout: "human log\nnot-json\n"}, nil + }} + _, err := testBackend(testConfig(), runner, io.Discard, io.Discard).Run(context.Background(), RunRequest{ + Repo: Repo{Name: "my-app", Root: newGitRepo(t)}, + Command: []string{"echo", "ok"}, + Env: map[string]string{"TOKEN": secret}, + }) + if err == nil || !strings.Contains(err.Error(), "protocol JSON response") { + t.Fatalf("Run err=%v", err) + } + if strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), "TOKEN") { + t.Fatalf("malformed response error leaked env: %v", err) + } +} + +func TestRunRedactsWorkflowFailure(t *testing.T) { + secret := "flue-secret-value" + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + return LocalCommandResult{ExitCode: 9, Stderr: "workflow saw " + secret}, errors.New("exit status 9") + }} + _, err := testBackend(testConfig(), runner, io.Discard, io.Discard).Run(context.Background(), RunRequest{ + Repo: Repo{Name: "my-app", Root: newGitRepo(t)}, + Command: []string{"echo", "ok"}, + Env: map[string]string{"TOKEN": secret}, + }) + var exitErr core.ExitError + if !core.AsExitError(err, &exitErr) || exitErr.Code != 9 { + t.Fatalf("Run err=%v", err) + } + if strings.Contains(err.Error(), secret) || !strings.Contains(err.Error(), "[REDACTED]") { + t.Fatalf("workflow failure was not redacted: %v", err) + } +} + +func TestRunClassifiesFlueTimeout(t *testing.T) { + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + return LocalCommandResult{}, context.DeadlineExceeded + }} + result, err := testBackend(testConfig(), runner, io.Discard, io.Discard).Run(context.Background(), RunRequest{ + Repo: Repo{Name: "my-app", Root: newGitRepo(t)}, + Command: []string{"sleep", "60"}, + }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("Run err=%v", err) + } + if result.Status != core.RunStatusTimedOut || result.ErrorKind != core.RunErrorTimeout { + t.Fatalf("result=%#v", result) + } +} + +func TestRunRejectsUnsupportedOptionsBeforeSpawn(t *testing.T) { + tests := []struct { + name string + cfg func(Config) Config + req RunRequest + want string + }{ + {name: "cloudflare target", cfg: func(cfg Config) Config { cfg.Flue.Target = "cloudflare"; return cfg }, req: RunRequest{Command: []string{"true"}}, want: "target=node only"}, + {name: "no sync", req: RunRequest{NoSync: true, Command: []string{"true"}}, want: "--no-sync is not supported"}, + {name: "sync only", req: RunRequest{SyncOnly: true, Command: []string{"true"}}, want: "--sync-only is not supported"}, + {name: "lease id", req: RunRequest{ID: "cbx_123", Command: []string{"true"}}, want: "persistent lease ids"}, + {name: "keep", req: RunRequest{Keep: true, Command: []string{"true"}}, want: "persistent lease ids"}, + {name: "desktop", req: RunRequest{Options: core.LeaseOptions{Desktop: true}, Command: []string{"true"}}, want: "desktop"}, + {name: "tailscale", req: RunRequest{Options: core.LeaseOptions{Tailscale: core.TailscaleConfig{Enabled: true}}, Command: []string{"true"}}, want: "Tailscale"}, + {name: "script", req: RunRequest{ScriptRequested: true, Command: []string{"true"}}, want: "--script is not supported"}, + {name: "fresh pr", req: RunRequest{FreshPR: core.FreshPRSpec{Owner: "example-org", Repo: "my-app", Number: 1}, Command: []string{"true"}}, want: "--fresh-pr is not supported"}, + {name: "proof", req: RunRequest{EmitProof: "proof.json", Command: []string{"true"}}, want: "--emit-proof is not supported"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := testConfig() + if tc.cfg != nil { + cfg = tc.cfg(cfg) + } + runner := &recordingRunner{} + req := tc.req + req.Repo = Repo{Name: "my-app", Root: newGitRepo(t)} + _, err := testBackend(cfg, runner, io.Discard, io.Discard).Run(context.Background(), req) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Run err=%v want %q", err, tc.want) + } + if len(runner.calls) != 0 { + t.Fatalf("runner called for rejected request: %#v", runner.calls) + } + }) + } +} + +func TestLifecycleIsOneShot(t *testing.T) { + backend := testBackend(testConfig(), &recordingRunner{}, io.Discard, io.Discard) + if err := backend.Warmup(context.Background(), WarmupRequest{}); err == nil || !strings.Contains(err.Error(), "one-shot") { + t.Fatalf("Warmup err=%v", err) + } + if leases, err := backend.List(context.Background(), ListRequest{}); err != nil || len(leases) != 0 { + t.Fatalf("List leases=%#v err=%v", leases, err) + } + if _, err := backend.Status(context.Background(), StatusRequest{}); err == nil || !strings.Contains(err.Error(), "one-shot") { + t.Fatalf("Status err=%v", err) + } + if err := backend.Stop(context.Background(), StopRequest{}); err == nil || !strings.Contains(err.Error(), "one-shot") { + t.Fatalf("Stop err=%v", err) + } +} + +type recordingRunner struct { + calls []LocalCommandRequest + fn func(context.Context, LocalCommandRequest) (LocalCommandResult, error) +} + +func (r *recordingRunner) Run(ctx context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + r.calls = append(r.calls, req) + if r.fn != nil { + return r.fn(ctx, req) + } + return LocalCommandResult{ExitCode: 0}, nil +} + +func (r *recordingRunner) onlyCall(t *testing.T) LocalCommandRequest { + t.Helper() + if len(r.calls) != 1 { + t.Fatalf("calls=%#v want one", r.calls) + } + return r.calls[0] +} + +func testConfig() Config { + cfg := core.BaseConfig() + cfg.Provider = providerName + cfg.Flue.CLIPath = "flue" + cfg.Flue.Workflow = defaultWorkflow + cfg.Flue.Target = defaultTarget + cfg.Flue.Workdir = defaultWorkdir + cfg.Flue.TimeoutSecs = defaultTimeoutSecs + return cfg +} + +func testBackend(cfg Config, runner *recordingRunner, stdout, stderr io.Writer) *backend { + rt := Runtime{Stdout: stdout, Stderr: stderr} + if runner != nil { + rt.Exec = runner + } + return &backend{spec: Provider{}.Spec(), cfg: cfg, rt: rt} +} + +func decodeCLIInputArg(t *testing.T, args []string) CLIInput { + t.Helper() + for i := 0; i < len(args)-1; i++ { + if args[i] == "--input" { + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(args[i+1]), &raw); err != nil { + t.Fatalf("parse --input: %v", err) + } + if len(raw) != 1 { + t.Fatalf("--input has keys=%v, want only requestFile", raw) + } + input, err := ParseCLIInput([]byte(args[i+1])) + if err != nil { + t.Fatalf("ParseCLIInput: %v", err) + } + return input + } + } + t.Fatalf("args missing --input: %#v", args) + return CLIInput{} +} + +func readProtocolRequest(t *testing.T, path string) Request { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read request: %v", err) + } + req, err := ParseRequest(data) + if err != nil { + t.Fatalf("parse request: %v", err) + } + return req +} + +func mustResponseJSON(t *testing.T, resp Response) string { + t.Helper() + data, err := json.Marshal(resp) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func newGitRepo(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("hello\n"), 0o644); err != nil { + t.Fatal(err) + } + runGit(t, dir, "init") + runGit(t, dir, "config", "user.email", "alice@example.com") + runGit(t, dir, "config", "user.name", "Alice") + runGit(t, dir, "add", "README.md") + runGit(t, dir, "commit", "-m", "init") + return dir +} + +func runGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} diff --git a/internal/providers/flue/cli.go b/internal/providers/flue/cli.go new file mode 100644 index 000000000..c92c59bdd --- /dev/null +++ b/internal/providers/flue/cli.go @@ -0,0 +1,160 @@ +package flue + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "strings" + "time" +) + +// Protocol stdout wraps delegated stdout/stderr inside JSON, so the process +// capture ceiling needs room for escaping and response framing. +const flueCapturedOutputLimitBytes = 1<<20 + 6*(defaultStdoutLimitBytes+defaultStderrLimitBytes) + +type flueCLI struct { + cfg FlueConfig + rt Runtime +} + +type flueRunResult struct { + Response Response + Raw LocalCommandResult +} + +func newFlueCLI(cfg Config, rt Runtime) (*flueCLI, error) { + if rt.Exec == nil { + return nil, exit(2, "provider=%s requires Runtime.Exec to invoke the Flue CLI", providerName) + } + if err := ValidateFlueConfig(cfg); err != nil { + return nil, err + } + if err := ValidateFlueRunTarget(cfg); err != nil { + return nil, err + } + return &flueCLI{cfg: cfg.Flue, rt: rt}, nil +} + +func (c *flueCLI) run(ctx context.Context, requestFile string, redactions []string) (flueRunResult, error) { + target := strings.ToLower(strings.TrimSpace(c.cfg.Target)) + if target == "" { + target = defaultTarget + } + if target != defaultTarget { + return flueRunResult{}, exit(2, "provider=%s supports flue target=node only in v1; upload/HTTP staging is required before %q can be used", providerName, target) + } + workflow := strings.TrimSpace(c.cfg.Workflow) + if workflow == "" { + return flueRunResult{}, exit(2, "flue workflow must not be empty") + } + input, err := json.Marshal(CLIInput{RequestFile: requestFile}) + if err != nil { + return flueRunResult{}, fmt.Errorf("encode flue request pointer: %w", err) + } + args := []string{"run", workflowSelector(workflow), "--target", target, "--input", string(input)} + root := cleanHostPath(c.cfg.Root) + if root != "" { + args = append(args, "--root", root) + } + if config := strings.TrimSpace(c.cfg.Config); config != "" { + args = append(args, "--config", config) + } + if envFile := strings.TrimSpace(c.cfg.EnvFile); envFile != "" { + args = append(args, "--env", envFile) + } + if output := strings.TrimSpace(c.cfg.Output); output != "" { + args = append(args, "--output", output) + } + runCtx := ctx + cancel := func() {} + if c.cfg.TimeoutSecs > 0 { + runCtx, cancel = context.WithTimeout(ctx, time.Duration(c.cfg.TimeoutSecs)*time.Second) + } + defer cancel() + result, runErr := c.rt.Exec.Run(runCtx, LocalCommandRequest{ + Name: blank(strings.TrimSpace(c.cfg.CLIPath), defaultCLIPath), + Args: args, + Dir: root, + MaxCapturedOutputBytes: flueCapturedOutputLimitBytes, + CancelGracePeriod: 5 * time.Second, + }) + if runErr != nil { + if errors.Is(runCtx.Err(), context.DeadlineExceeded) || errors.Is(runErr, context.DeadlineExceeded) { + return flueRunResult{Raw: result}, context.DeadlineExceeded + } + if errors.Is(runCtx.Err(), context.Canceled) || errors.Is(runErr, context.Canceled) { + return flueRunResult{Raw: result}, context.Canceled + } + } + resp, err := ParseResponseFromStdout(result.Stdout) + if err != nil { + if runErr != nil { + return flueRunResult{Raw: result}, exit(flueProcessExitCode(result.ExitCode), "flue workflow failed: %s", flueFailureDetail(result, runErr, redactions)) + } + return flueRunResult{Raw: result}, err + } + if runErr != nil && resp.ExitCode == 0 { + return flueRunResult{Response: resp, Raw: result}, exit(flueProcessExitCode(result.ExitCode), "flue workflow failed: %s", flueFailureDetail(result, runErr, redactions)) + } + return flueRunResult{Response: resp, Raw: result}, nil +} + +func workflowSelector(workflow string) string { + workflow = strings.TrimSpace(workflow) + if strings.HasPrefix(workflow, "workflow:") { + return workflow + } + return "workflow:" + workflow +} + +func flueProcessExitCode(code int) int { + if code != 0 { + return code + } + return 1 +} + +func flueFailureDetail(result LocalCommandResult, err error, redactions []string) string { + detail := strings.TrimSpace(result.Stderr) + if detail == "" { + detail = strings.TrimSpace(result.Stdout) + } + if detail == "" && err != nil { + detail = err.Error() + } + if detail == "" { + detail = "unknown failure" + } + return redactFlueDetail(tailString(detail, 4096), redactions) +} + +func redactFlueDetail(value string, redactions []string) string { + out := value + for _, secret := range redactions { + secret = strings.TrimSpace(secret) + if secret == "" { + continue + } + out = strings.ReplaceAll(out, secret, "[REDACTED]") + } + return out +} + +func tailString(value string, limit int) string { + if limit <= 0 || len(value) <= limit { + return value + } + return value[len(value)-limit:] +} + +func cleanHostPath(value string) string { + if strings.TrimSpace(value) == "" { + return "" + } + if abs, err := filepath.Abs(value); err == nil { + return abs + } + return value +} diff --git a/internal/providers/flue/cli_test.go b/internal/providers/flue/cli_test.go new file mode 100644 index 000000000..69b4dc033 --- /dev/null +++ b/internal/providers/flue/cli_test.go @@ -0,0 +1,106 @@ +package flue + +import ( + "context" + "io" + "path/filepath" + "reflect" + "strings" + "testing" + + core "github.com/openclaw/crabbox/internal/cli" +) + +func TestCLIAdapterBuildsFlueRunArgs(t *testing.T) { + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + input := decodeCLIInputArg(t, req.Args) + if input.RequestFile != "/tmp/request.json" { + t.Fatalf("requestFile=%q", input.RequestFile) + } + return LocalCommandResult{ExitCode: 0, Stdout: mustResponseJSON(t, Response{ProtocolVersion: protocolVersion, Operation: operationRun, ExitCode: 0})}, nil + }} + cfg := core.BaseConfig() + cfg.Provider = providerName + cfg.Flue = FlueConfig{ + CLIPath: "/opt/flue/bin/flue", + Root: "/tmp/flue-project", + Workflow: "workflow:runner", + Target: "node", + Config: "/tmp/flue.config.ts", + EnvFile: "/tmp/flue.env", + Output: "json", + Workdir: defaultWorkdir, + TimeoutSecs: 10, + } + cli, err := newFlueCLI(cfg, Runtime{Exec: runner, Stdout: io.Discard, Stderr: io.Discard}) + if err != nil { + t.Fatal(err) + } + if _, err := cli.run(context.Background(), "/tmp/request.json", nil); err != nil { + t.Fatalf("run err=%v", err) + } + call := runner.onlyCall(t) + if call.Name != "/opt/flue/bin/flue" || call.Dir != "/tmp/flue-project" { + t.Fatalf("call name/dir=%q/%q", call.Name, call.Dir) + } + want := []string{ + "run", "workflow:runner", "--target", "node", "--input", `{"requestFile":"/tmp/request.json"}`, + "--root", "/tmp/flue-project", + "--config", "/tmp/flue.config.ts", + "--env", "/tmp/flue.env", + "--output", "json", + } + if !reflect.DeepEqual(call.Args, want) { + t.Fatalf("args=%#v want %#v", call.Args, want) + } + rawOutputLimit := defaultStdoutLimitBytes + defaultStderrLimitBytes + if call.MaxCapturedOutputBytes <= rawOutputLimit || call.MaxCapturedOutputBytes < 6*rawOutputLimit { + t.Fatalf("MaxCapturedOutputBytes=%d too small for JSON protocol output limit %d", call.MaxCapturedOutputBytes, rawOutputLimit) + } +} + +func TestCLIAdapterNormalizesRelativeRoot(t *testing.T) { + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + return LocalCommandResult{ExitCode: 0, Stdout: mustResponseJSON(t, Response{ProtocolVersion: protocolVersion, Operation: operationRun, ExitCode: 0})}, nil + }} + cfg := testConfig() + cfg.Flue.Root = "flue-runner" + cli, err := newFlueCLI(cfg, Runtime{Exec: runner, Stdout: io.Discard, Stderr: io.Discard}) + if err != nil { + t.Fatal(err) + } + if _, err := cli.run(context.Background(), "/tmp/request.json", nil); err != nil { + t.Fatalf("run err=%v", err) + } + call := runner.onlyCall(t) + wantRoot, err := filepath.Abs("flue-runner") + if err != nil { + t.Fatal(err) + } + if call.Dir != wantRoot { + t.Fatalf("Dir=%q want %q", call.Dir, wantRoot) + } + inputIndex := -1 + for i, arg := range call.Args { + if arg == "--root" { + inputIndex = i + break + } + } + if inputIndex < 0 || inputIndex+1 >= len(call.Args) || call.Args[inputIndex+1] != wantRoot { + t.Fatalf("args=%#v want --root %q", call.Args, wantRoot) + } +} + +func TestCLIAdapterRejectsUnsupportedTargetBeforeSpawn(t *testing.T) { + cfg := testConfig() + cfg.Flue.Target = "cloudflare" + runner := &recordingRunner{} + _, err := newFlueCLI(cfg, Runtime{Exec: runner, Stdout: io.Discard, Stderr: io.Discard}) + if err == nil || !strings.Contains(err.Error(), "target=node only") { + t.Fatalf("newFlueCLI err=%v", err) + } + if len(runner.calls) != 0 { + t.Fatalf("runner called: %#v", runner.calls) + } +} diff --git a/internal/providers/flue/core.go b/internal/providers/flue/core.go new file mode 100644 index 000000000..65b8ee3b9 --- /dev/null +++ b/internal/providers/flue/core.go @@ -0,0 +1,125 @@ +package flue + +import ( + "context" + "flag" + "io" + "os" + "time" + + core "github.com/openclaw/crabbox/internal/cli" +) + +type Config = core.Config +type FlueConfig = core.FlueConfig +type ProviderSpec = core.ProviderSpec +type Runtime = core.Runtime +type Backend = core.Backend +type DoctorRequest = core.DoctorRequest +type DoctorResult = core.DoctorResult +type DoctorCheck = core.DoctorCheck +type WarmupRequest = core.WarmupRequest +type RunRequest = core.RunRequest +type RunResult = core.RunResult +type ListRequest = core.ListRequest +type LeaseView = core.LeaseView +type StatusRequest = core.StatusRequest +type StatusView = core.StatusView +type StopRequest = core.StopRequest +type Repo = core.Repo +type SyncManifest = core.SyncManifest +type LocalCommandRequest = core.LocalCommandRequest +type LocalCommandResult = core.LocalCommandResult +type timingReport = core.TimingReport +type timingPhase = core.TimingPhase + +const ( + providerName = "flue" + providerKind = "delegated-run" + defaultCLIPath = "flue" + defaultWorkflow = "crabbox-runner" + defaultTarget = "node" + defaultWorkdir = "/workspace/crabbox" + defaultTimeoutSecs = 1800 + protocolVersion = 1 + operationRun = "run" +) + +func exit(code int, format string, args ...any) core.ExitError { + return core.Exit(code, format, args...) +} + +func flagWasSet(fs *flag.FlagSet, name string) bool { + return core.FlagWasSet(fs, name) +} + +func blank(value, fallback string) string { + return core.Blank(value, fallback) +} + +func newLeaseID() string { + return core.NewLeaseID() +} + +func newLeaseSlug(leaseID string) string { + return core.NewLeaseSlug(leaseID) +} + +func normalizeLeaseSlug(value string) string { + return core.NormalizeLeaseSlug(value) +} + +func writeTimingJSON(w io.Writer, report timingReport) error { + return core.WriteTimingJSON(w, report) +} + +func timingReportWithRunResult(report timingReport, result RunResult, err error) timingReport { + return core.TimingReportWithRunResult(report, result, err) +} + +func printEnvForwardingSummary(w io.Writer, provider, behavior string, allow []string, env map[string]string) { + core.PrintEnvForwardingSummary(w, provider, behavior, allow, env) +} + +func shellScriptFromArgv(command []string) string { + return core.ShellScriptFromArgv(command) +} + +func shouldUseShell(command []string) bool { + return core.ShouldUseShell(command) +} + +func leadingEnvAssignment(command []string) bool { + return core.LeadingEnvAssignment(command) +} + +func syncExcludes(root string, cfg Config) ([]string, error) { + return core.SyncExcludes(root, cfg) +} + +func syncManifest(root string, excludes, includes []string) (SyncManifest, error) { + return core.BuildSyncManifestFiltered(root, excludes, includes) +} + +func checkSyncPreflight(manifest SyncManifest, cfg Config, force bool, stderr io.Writer) error { + return core.CheckSyncPreflight(manifest, cfg, force, stderr) +} + +func createPortableSyncArchive(ctx context.Context, repo Repo, manifest SyncManifest, tempPattern string) (*os.File, error) { + return core.CreateSyncArchive(ctx, repo, manifest, tempPattern) +} + +func finalizeRunResult(result RunResult, err error) RunResult { + return core.FinalizeRunResult(result, err) +} + +func rejectDelegatedSyncOptionsForSpec(spec ProviderSpec, req RunRequest) error { + return core.RejectDelegatedSyncOptionsForSpec(spec, req) +} + +func durationMillis(duration time.Duration) int64 { + if duration <= 0 { + return 0 + } + return duration.Milliseconds() +} diff --git a/internal/providers/flue/doctor.go b/internal/providers/flue/doctor.go new file mode 100644 index 000000000..6ca85ce50 --- /dev/null +++ b/internal/providers/flue/doctor.go @@ -0,0 +1,263 @@ +package flue + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +const flueDoctorOutputLimitBytes = 64 * 1024 + +func (b *backend) Doctor(ctx context.Context, _ DoctorRequest) (DoctorResult, error) { + result := DoctorResult{Provider: providerName} + checks := []DoctorCheck{ + b.doctorRootCheck(), + b.doctorOptionalFileCheck("config", b.cfg.Flue.Config), + b.doctorOptionalFileCheck("env_file", b.cfg.Flue.EnvFile), + b.doctorOutputCheck(), + b.doctorTargetCheck(), + b.doctorWorkflowCheck(), + } + help, helpErr := b.doctorRun(ctx, []string{"--help"}) + checks = append([]DoctorCheck{doctorCLIHelpCheck(b.cfg, help, helpErr)}, checks...) + version, versionErr := b.doctorRun(ctx, []string{"--version"}) + checks = append(checks, doctorCLIVersionCheck(version, versionErr)) + result.Checks = checks + if doctorChecksFailed(checks) { + result.Status = "error" + result.Message = "cli=blocked target=" + flueDoctorTarget(b.cfg) + " workflow=" + flueDoctorWorkflow(b.cfg) + " mutation=false" + return result, nil + } + result.Status = "ok" + result.Message = "cli=ready target=node workflow=" + flueDoctorWorkflow(b.cfg) + " workflow_discovery=unchecked mutation=false" + return result, nil +} + +func (b *backend) doctorRun(ctx context.Context, args []string) (LocalCommandResult, error) { + if b.rt.Exec == nil { + return LocalCommandResult{}, exit(2, "provider=%s requires Runtime.Exec to invoke the Flue CLI", providerName) + } + return b.rt.Exec.Run(ctx, LocalCommandRequest{ + Name: flueDoctorCLIPath(b.cfg), + Args: args, + Dir: flueDoctorCommandDir(b.cfg), + MaxCapturedOutputBytes: flueDoctorOutputLimitBytes, + CancelGracePeriod: 2 * time.Second, + }) +} + +func (b *backend) doctorRootCheck() DoctorCheck { + root := strings.TrimSpace(b.cfg.Flue.Root) + details := map[string]string{"mutation": "false"} + if root == "" { + cwd, err := os.Getwd() + if err != nil { + return DoctorCheck{Status: "warning", Check: "flue_root", Message: "root unset; current working directory could not be resolved", Details: details} + } + details["mode"] = "cwd" + details["path"] = cwd + return DoctorCheck{Status: "ok", Check: "flue_root", Message: "root unset; Flue will use the current working directory", Details: details} + } + path := flueDoctorResolvePath("", root) + details["path"] = path + info, err := os.Stat(path) + if err != nil { + return DoctorCheck{Status: "failed", Check: "flue_root", Message: "configured Flue root is not readable: " + err.Error(), Details: details} + } + if !info.IsDir() { + return DoctorCheck{Status: "failed", Check: "flue_root", Message: "configured Flue root is not a directory", Details: details} + } + return DoctorCheck{Status: "ok", Check: "flue_root", Message: "ready", Details: details} +} + +func (b *backend) doctorOptionalFileCheck(name, value string) DoctorCheck { + details := map[string]string{"mutation": "false"} + value = strings.TrimSpace(value) + if value == "" { + details["configured"] = "false" + return DoctorCheck{Status: "ok", Check: name, Message: "not configured", Details: details} + } + path := flueDoctorResolvePath(b.cfg.Flue.Root, value) + details["configured"] = "true" + details["path"] = path + info, err := os.Stat(path) + if err != nil { + return DoctorCheck{Status: "failed", Check: name, Message: "configured path is not readable: " + err.Error(), Details: details} + } + if info.IsDir() { + return DoctorCheck{Status: "failed", Check: name, Message: "configured path is a directory; expected a file", Details: details} + } + file, err := os.Open(path) + if err != nil { + return DoctorCheck{Status: "failed", Check: name, Message: "configured path is not readable: " + err.Error(), Details: details} + } + _ = file.Close() + return DoctorCheck{Status: "ok", Check: name, Message: "ready", Details: details} +} + +func (b *backend) doctorOutputCheck() DoctorCheck { + output := strings.TrimSpace(b.cfg.Flue.Output) + details := map[string]string{"mutation": "false"} + if output == "" { + details["configured"] = "false" + return DoctorCheck{Status: "ok", Check: "output", Message: "using Flue default output mode", Details: details} + } + details["configured"] = "true" + details["value"] = output + if strings.ContainsRune(output, '\x00') { + return DoctorCheck{Status: "failed", Check: "output", Message: "output mode contains a NUL byte", Details: details} + } + return DoctorCheck{Status: "ok", Check: "output", Message: "configured", Details: details} +} + +func (b *backend) doctorTargetCheck() DoctorCheck { + target := flueDoctorTarget(b.cfg) + details := map[string]string{"target": target, "mutation": "false"} + if target != defaultTarget { + return DoctorCheck{ + Status: "failed", + Check: "target", + Message: fmt.Sprintf("provider=%s supports flue target=node only in v1; upload/HTTP staging is required before %q can be used", providerName, target), + Details: details, + } + } + return DoctorCheck{Status: "ok", Check: "target", Message: "node", Details: details} +} + +func (b *backend) doctorWorkflowCheck() DoctorCheck { + workflow := flueDoctorWorkflow(b.cfg) + details := map[string]string{ + "workflow": workflow, + "discoverability": "unchecked", + "reason": "safe_flue_discovery_unavailable", + "mutation": "false", + } + if workflow == "" { + return DoctorCheck{Status: "failed", Check: "workflow", Message: "flue workflow must not be empty", Details: details} + } + return DoctorCheck{ + Status: "warning", + Check: "workflow", + Message: "workflow existence is unchecked because running Flue workflows may execute user code", + Details: details, + } +} + +func doctorCLIHelpCheck(cfg Config, result LocalCommandResult, err error) DoctorCheck { + details := map[string]string{ + "cli": flueDoctorCLIPath(cfg), + "mutation": "false", + } + if root := strings.TrimSpace(cfg.Flue.Root); root != "" { + details["root"] = flueDoctorResolvePath("", root) + } + if err != nil || result.ExitCode != 0 { + return DoctorCheck{Status: "failed", Check: "flue_help", Message: flueDoctorFailure(result, err), Details: details} + } + if line := firstNonEmptyLine(result.Stdout); line != "" { + details["help"] = line + } + return DoctorCheck{Status: "ok", Check: "flue_help", Message: "ready", Details: details} +} + +func doctorCLIVersionCheck(result LocalCommandResult, err error) DoctorCheck { + details := map[string]string{"authoritative": "false", "optional": "true", "mutation": "false"} + if err != nil || result.ExitCode != 0 { + return DoctorCheck{Status: "warning", Check: "flue_version", Message: flueDoctorFailure(result, err), Details: details} + } + version := firstNonEmptyLine(result.Stdout) + if version == "" { + version = strings.TrimSpace(result.Stderr) + } + if version == "" { + version = "reported" + } + details["version"] = version + return DoctorCheck{Status: "ok", Check: "flue_version", Message: "version reported; compatibility is based on command-surface checks", Details: details} +} + +func flueDoctorFailure(result LocalCommandResult, err error) string { + detail := strings.TrimSpace(result.Stderr) + if detail == "" { + detail = strings.TrimSpace(result.Stdout) + } + if detail == "" && err != nil { + detail = err.Error() + } + if detail == "" { + detail = "command failed" + } + if err != nil && !errors.Is(err, context.DeadlineExceeded) && !strings.Contains(detail, err.Error()) { + detail += ": " + err.Error() + } + return tailString(detail, 4096) +} + +func flueDoctorResolvePath(root, value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if filepath.IsAbs(value) { + if abs, err := filepath.Abs(value); err == nil { + return abs + } + return value + } + if strings.TrimSpace(root) != "" { + value = filepath.Join(root, value) + } + if abs, err := filepath.Abs(value); err == nil { + return abs + } + return value +} + +func flueDoctorCLIPath(cfg Config) string { + return blank(strings.TrimSpace(cfg.Flue.CLIPath), defaultCLIPath) +} + +func flueDoctorCommandDir(cfg Config) string { + root := strings.TrimSpace(cfg.Flue.Root) + if root == "" { + return "" + } + path := flueDoctorResolvePath("", root) + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + return "" + } + return path +} + +func flueDoctorTarget(cfg Config) string { + return strings.ToLower(blank(strings.TrimSpace(cfg.Flue.Target), defaultTarget)) +} + +func flueDoctorWorkflow(cfg Config) string { + return strings.TrimSpace(cfg.Flue.Workflow) +} + +func doctorChecksFailed(checks []DoctorCheck) bool { + for _, check := range checks { + switch strings.ToLower(strings.TrimSpace(check.Status)) { + case "failed", "missing", "error": + return true + } + } + return false +} + +func firstNonEmptyLine(value string) string { + for _, line := range strings.Split(value, "\n") { + line = strings.TrimSpace(line) + if line != "" { + return line + } + } + return "" +} diff --git a/internal/providers/flue/doctor_test.go b/internal/providers/flue/doctor_test.go new file mode 100644 index 000000000..9337debc0 --- /dev/null +++ b/internal/providers/flue/doctor_test.go @@ -0,0 +1,185 @@ +package flue + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestDoctorChecksLocalFlueSurfaceAndPathsWithoutRunningWorkflow(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "flue.config.ts"), []byte("export default {}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, ".env"), []byte("TOKEN=redacted\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg := testConfig() + cfg.Flue.Root = root + cfg.Flue.Config = "flue.config.ts" + cfg.Flue.EnvFile = ".env" + cfg.Flue.Output = "json" + cfg.Flue.CLIPath = "/opt/flue/bin/flue" + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + switch strings.Join(req.Args, " ") { + case "--help": + if req.Dir != root { + t.Fatalf("help Dir=%q want %q", req.Dir, root) + } + return LocalCommandResult{ExitCode: 0, Stdout: "Usage: flue run workflow:name --input \n"}, nil + case "--version": + return LocalCommandResult{ExitCode: 1, Stderr: "version unavailable"}, errors.New("version unavailable") + default: + t.Fatalf("doctor must not run workflow, got args=%v", req.Args) + return LocalCommandResult{}, nil + } + }} + result, err := testBackend(cfg, runner, io.Discard, io.Discard).Doctor(context.Background(), DoctorRequest{}) + if err != nil { + t.Fatalf("Doctor err=%v", err) + } + if result.Status != "ok" || !strings.Contains(result.Message, "workflow_discovery=unchecked") { + t.Fatalf("result=%#v", result) + } + if len(runner.calls) != 2 { + t.Fatalf("runner calls=%#v want help and version only", runner.calls) + } + assertDoctorCheck(t, result, "flue_help", "ok") + assertDoctorCheck(t, result, "flue_root", "ok") + assertDoctorCheck(t, result, "config", "ok") + assertDoctorCheck(t, result, "env_file", "ok") + assertDoctorCheck(t, result, "output", "ok") + assertDoctorCheck(t, result, "target", "ok") + workflow := assertDoctorCheck(t, result, "workflow", "warning") + if workflow.Details["discoverability"] != "unchecked" || workflow.Details["mutation"] != "false" { + t.Fatalf("workflow check=%#v", workflow) + } + version := assertDoctorCheck(t, result, "flue_version", "warning") + if version.Details["optional"] != "true" { + t.Fatalf("version check=%#v", version) + } +} + +func TestDoctorReportsMissingRootAndUnsupportedTarget(t *testing.T) { + cfg := testConfig() + cfg.Flue.Root = filepath.Join(t.TempDir(), "missing") + cfg.Flue.Target = "cloudflare" + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + if req.Dir != "" { + t.Fatalf("doctor should not use missing root as command dir, got %q", req.Dir) + } + switch strings.Join(req.Args, " ") { + case "--help": + return LocalCommandResult{ExitCode: 0, Stdout: "Usage: flue\n"}, nil + case "--version": + return LocalCommandResult{ExitCode: 0, Stdout: "flue 1.2.3\n"}, nil + default: + t.Fatalf("unexpected args=%v", req.Args) + return LocalCommandResult{}, nil + } + }} + result, err := testBackend(cfg, runner, io.Discard, io.Discard).Doctor(context.Background(), DoctorRequest{}) + if err != nil { + t.Fatalf("Doctor err=%v", err) + } + if result.Status != "error" { + t.Fatalf("result=%#v", result) + } + root := assertDoctorCheck(t, result, "flue_root", "failed") + if root.Details["path"] == "" { + t.Fatalf("root details=%#v", root.Details) + } + target := assertDoctorCheck(t, result, "target", "failed") + if !strings.Contains(target.Message, "target=node only") { + t.Fatalf("target check=%#v", target) + } +} + +func TestDoctorReportsUnreadableConfiguredFile(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("chmod unreadable fixture is Unix-specific") + } + root := t.TempDir() + config := filepath.Join(root, "flue.config.ts") + if err := os.WriteFile(config, []byte("export default {}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(config, 0); err != nil { + t.Fatal(err) + } + defer func() { _ = os.Chmod(config, 0o600) }() + if file, err := os.Open(config); err == nil { + _ = file.Close() + t.Skip("current user can read chmod 000 files") + } + + cfg := testConfig() + cfg.Flue.Root = root + cfg.Flue.Config = "flue.config.ts" + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + switch strings.Join(req.Args, " ") { + case "--help": + return LocalCommandResult{ExitCode: 0, Stdout: "Usage: flue\n"}, nil + case "--version": + return LocalCommandResult{ExitCode: 0, Stdout: "flue 1.2.3\n"}, nil + default: + t.Fatalf("unexpected args=%v", req.Args) + return LocalCommandResult{}, nil + } + }} + result, err := testBackend(cfg, runner, io.Discard, io.Discard).Doctor(context.Background(), DoctorRequest{}) + if err != nil { + t.Fatalf("Doctor err=%v", err) + } + if result.Status != "error" { + t.Fatalf("result=%#v", result) + } + configCheck := assertDoctorCheck(t, result, "config", "failed") + if !strings.Contains(configCheck.Message, "not readable") { + t.Fatalf("config check=%#v", configCheck) + } +} + +func TestDoctorReportsMissingCLIWithoutMutation(t *testing.T) { + cfg := testConfig() + runner := &recordingRunner{fn: func(_ context.Context, req LocalCommandRequest) (LocalCommandResult, error) { + if len(req.Args) == 1 && req.Args[0] == "--help" { + return LocalCommandResult{ExitCode: 127, Stderr: "flue not found"}, errors.New("not found") + } + if len(req.Args) == 1 && req.Args[0] == "--version" { + return LocalCommandResult{ExitCode: 127, Stderr: "flue not found"}, errors.New("not found") + } + t.Fatalf("unexpected args=%v", req.Args) + return LocalCommandResult{}, nil + }} + result, err := testBackend(cfg, runner, io.Discard, io.Discard).Doctor(context.Background(), DoctorRequest{}) + if err != nil { + t.Fatalf("Doctor err=%v", err) + } + if result.Status != "error" { + t.Fatalf("result=%#v", result) + } + help := assertDoctorCheck(t, result, "flue_help", "failed") + if !strings.Contains(help.Message, "flue not found") || help.Details["mutation"] != "false" { + t.Fatalf("help check=%#v", help) + } +} + +func assertDoctorCheck(t *testing.T, result DoctorResult, name, status string) DoctorCheck { + t.Helper() + for _, check := range result.Checks { + if check.Check == name { + if check.Status != status { + t.Fatalf("check %s status=%q want %q: %#v", name, check.Status, status, check) + } + return check + } + } + t.Fatalf("missing doctor check %q in %#v", name, result.Checks) + return DoctorCheck{} +} diff --git a/internal/providers/flue/flags.go b/internal/providers/flue/flags.go new file mode 100644 index 000000000..430433961 --- /dev/null +++ b/internal/providers/flue/flags.go @@ -0,0 +1,126 @@ +package flue + +import ( + "flag" + "path" + "strings" +) + +type flueFlagValues struct { + CLIPath *string + Root *string + Workflow *string + Target *string + Config *string + EnvFile *string + Output *string + Workdir *string + TimeoutSecs *int +} + +func RegisterFlueProviderFlags(fs *flag.FlagSet, defaults Config) any { + return flueFlagValues{ + CLIPath: fs.String("flue-cli", defaults.Flue.CLIPath, "Path to the flue CLI binary"), + Root: fs.String("flue-root", defaults.Flue.Root, "Flue project root directory"), + Workflow: fs.String("flue-workflow", defaults.Flue.Workflow, "Flue workflow name for Crabbox delegated runs"), + Target: fs.String("flue-target", defaults.Flue.Target, "Flue run target (v1 supports node only)"), + Config: fs.String("flue-config", defaults.Flue.Config, "Flue config file path"), + EnvFile: fs.String("flue-env", defaults.Flue.EnvFile, "Flue env file path; secrets stay in this file or Flue-managed env"), + Output: fs.String("flue-output", defaults.Flue.Output, "Flue output mode"), + Workdir: fs.String("flue-workdir", defaults.Flue.Workdir, "Absolute working directory inside the Flue sandbox"), + TimeoutSecs: fs.Int("flue-timeout-secs", defaults.Flue.TimeoutSecs, "Flue delegated run timeout in seconds"), + } +} + +func ApplyFlueProviderFlags(cfg *Config, fs *flag.FlagSet, values any) error { + if strings.EqualFold(strings.TrimSpace(cfg.Provider), providerName) { + if flagWasSet(fs, "class") { + return exit(2, "--class is not supported for provider=%s; configure the Flue workflow instead", providerName) + } + if flagWasSet(fs, "type") { + return exit(2, "--type is not supported for provider=%s; configure the Flue workflow instead", providerName) + } + } + v, ok := values.(flueFlagValues) + if !ok { + return nil + } + if flagWasSet(fs, "flue-cli") { + cfg.Flue.CLIPath = *v.CLIPath + } + if flagWasSet(fs, "flue-root") { + cfg.Flue.Root = *v.Root + } + if flagWasSet(fs, "flue-workflow") { + cfg.Flue.Workflow = *v.Workflow + } + if flagWasSet(fs, "flue-target") { + cfg.Flue.Target = *v.Target + } + if flagWasSet(fs, "flue-config") { + cfg.Flue.Config = *v.Config + } + if flagWasSet(fs, "flue-env") { + cfg.Flue.EnvFile = *v.EnvFile + } + if flagWasSet(fs, "flue-output") { + cfg.Flue.Output = *v.Output + } + if flagWasSet(fs, "flue-workdir") { + cfg.Flue.Workdir = *v.Workdir + } + if flagWasSet(fs, "flue-timeout-secs") { + cfg.Flue.TimeoutSecs = *v.TimeoutSecs + } + return ValidateFlueDoctorConfig(*cfg) +} + +func ValidateFlueConfig(cfg Config) error { + if err := ValidateFlueDoctorConfig(cfg); err != nil { + return err + } + return ValidateFlueRunTarget(cfg) +} + +func ValidateFlueDoctorConfig(cfg Config) error { + if strings.TrimSpace(cfg.Flue.CLIPath) == "" { + return exit(2, "flue cliPath must not be empty") + } + if strings.TrimSpace(cfg.Flue.Workflow) == "" { + return exit(2, "flue workflow must not be empty") + } + if cfg.Flue.TimeoutSecs < 0 { + return exit(2, "flue timeoutSecs must be non-negative") + } + if _, err := cleanWorkdir(cfg.Flue.Workdir); err != nil { + return err + } + return nil +} + +func ValidateFlueRunTarget(cfg Config) error { + target := strings.ToLower(strings.TrimSpace(cfg.Flue.Target)) + if target == "" { + target = defaultTarget + } + if target != defaultTarget { + return exit(2, "provider=%s supports flue target=node only in v1; upload/HTTP staging is required before %q can be used", providerName, target) + } + return nil +} + +func cleanWorkdir(value string) (string, error) { + workdir := strings.TrimSpace(value) + if workdir == "" { + workdir = defaultWorkdir + } + if !strings.HasPrefix(workdir, "/") { + return "", exit(2, "flue workdir %q must be absolute", workdir) + } + clean := path.Clean(workdir) + switch clean { + case "/", "/tmp", "/home", "/workspace": + return "", exit(2, "flue workdir %q is too broad; use a provider-owned subdirectory such as %s", clean, defaultWorkdir) + } + return clean, nil +} diff --git a/internal/providers/flue/protocol.go b/internal/providers/flue/protocol.go new file mode 100644 index 000000000..d2e2f148b --- /dev/null +++ b/internal/providers/flue/protocol.go @@ -0,0 +1,201 @@ +package flue + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +const ( + defaultStdoutLimitBytes = 10 * 1024 * 1024 + defaultStderrLimitBytes = 10 * 1024 * 1024 + maxArtifactMetadataItems = 128 +) + +type CLIInput struct { + RequestFile string `json:"requestFile"` +} + +type Request struct { + ProtocolVersion int `json:"protocolVersion"` + Operation string `json:"operation"` + LeaseID string `json:"leaseId,omitempty"` + Slug string `json:"slug,omitempty"` + Workflow string `json:"workflow"` + Target string `json:"target"` + WorkspaceArchive string `json:"workspaceArchive"` + Workspace string `json:"workspace"` + Command []string `json:"command"` + Env map[string]string `json:"env,omitempty"` + TimeoutMs int64 `json:"timeoutMs,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + OutputLimits OutputLimits `json:"outputLimits,omitempty"` +} + +type OutputLimits struct { + StdoutBytes int64 `json:"stdoutBytes,omitempty"` + StderrBytes int64 `json:"stderrBytes,omitempty"` +} + +type Response struct { + ProtocolVersion int `json:"protocolVersion"` + Operation string `json:"operation,omitempty"` + LeaseID string `json:"leaseId,omitempty"` + Slug string `json:"slug,omitempty"` + ExitCode int `json:"exitCode"` + Stdout string `json:"stdout,omitempty"` + Stderr string `json:"stderr,omitempty"` + Timing ResponseTiming `json:"timing,omitempty"` + Artifacts []ResponseArtifact `json:"artifacts,omitempty"` + Error string `json:"error,omitempty"` +} + +type ResponseTiming struct { + TotalMs int64 `json:"totalMs,omitempty"` + RunMs int64 `json:"runMs,omitempty"` +} + +type ResponseArtifact struct { + Path string `json:"path"` + Destination string `json:"destination,omitempty"` + SizeBytes int64 `json:"sizeBytes,omitempty"` + SHA256 string `json:"sha256,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +func ParseCLIInput(data []byte) (CLIInput, error) { + var input CLIInput + if err := json.Unmarshal(data, &input); err != nil { + return CLIInput{}, fmt.Errorf("parse flue input pointer: malformed JSON") + } + if err := input.Validate(); err != nil { + return CLIInput{}, err + } + return input, nil +} + +func (input CLIInput) Validate() error { + if strings.TrimSpace(input.RequestFile) == "" { + return exit(2, "flue input requestFile is required") + } + return nil +} + +func ParseRequest(data []byte) (Request, error) { + var req Request + if err := json.Unmarshal(data, &req); err != nil { + return Request{}, fmt.Errorf("parse flue request: malformed JSON") + } + if err := req.Validate(); err != nil { + return Request{}, err + } + return req, nil +} + +func (req Request) Validate() error { + if req.ProtocolVersion != protocolVersion { + return exit(2, "unsupported flue protocolVersion %d", req.ProtocolVersion) + } + if strings.TrimSpace(req.Operation) != operationRun { + return exit(2, "unsupported flue operation %q", strings.TrimSpace(req.Operation)) + } + if strings.TrimSpace(req.Workflow) == "" { + return exit(2, "flue request workflow is required") + } + target := strings.TrimSpace(req.Target) + if target == "" { + target = defaultTarget + } + if target != defaultTarget { + return exit(2, "flue request target %q is unsupported in v1", target) + } + if strings.TrimSpace(req.WorkspaceArchive) == "" { + return exit(2, "flue request workspaceArchive is required") + } + if strings.TrimSpace(req.Workspace) == "" { + return exit(2, "flue request workspace is required") + } + if len(req.Command) == 0 { + return exit(2, "flue request command is required") + } + if strings.TrimSpace(req.Command[0]) == "" { + return exit(2, "flue request command[0] must not be empty") + } + if req.TimeoutMs < 0 { + return exit(2, "flue request timeoutMs must be non-negative") + } + if req.OutputLimits.StdoutBytes < 0 || req.OutputLimits.StderrBytes < 0 { + return exit(2, "flue request output limits must be non-negative") + } + return nil +} + +func (req Request) EffectiveOutputLimits() OutputLimits { + limits := req.OutputLimits + if limits.StdoutBytes == 0 { + limits.StdoutBytes = defaultStdoutLimitBytes + } + if limits.StderrBytes == 0 { + limits.StderrBytes = defaultStderrLimitBytes + } + return limits +} + +func ParseResponse(data []byte) (Response, error) { + var resp Response + if err := json.Unmarshal(data, &resp); err != nil { + return Response{}, fmt.Errorf("parse flue response: malformed JSON") + } + if err := resp.Validate(); err != nil { + return Response{}, err + } + return resp, nil +} + +func (resp Response) Validate() error { + if resp.ProtocolVersion != protocolVersion { + return exit(2, "unsupported flue response protocolVersion %d", resp.ProtocolVersion) + } + if op := strings.TrimSpace(resp.Operation); op != "" && op != operationRun { + return exit(2, "unsupported flue response operation %q", op) + } + if resp.Timing.TotalMs < 0 || resp.Timing.RunMs < 0 { + return exit(2, "flue response timing must be non-negative") + } + if len(resp.Artifacts) > maxArtifactMetadataItems { + return exit(2, "flue response artifacts exceed limit %d", maxArtifactMetadataItems) + } + for i, artifact := range resp.Artifacts { + if strings.TrimSpace(artifact.Path) == "" { + return exit(2, "flue response artifact[%d] path is required", i) + } + if artifact.SizeBytes < 0 { + return exit(2, "flue response artifact[%d] sizeBytes must be non-negative", i) + } + } + return nil +} + +func ParseResponseFromStdout(stdout string) (Response, error) { + trimmed := strings.TrimSpace(stdout) + if trimmed == "" { + return Response{}, exit(5, "flue run produced no protocol response on stdout") + } + if resp, err := ParseResponse([]byte(trimmed)); err == nil { + return resp, nil + } + lines := bytes.Split([]byte(stdout), []byte{'\n'}) + for i := len(lines) - 1; i >= 0; i-- { + line := bytes.TrimSpace(lines[i]) + if len(line) == 0 || line[0] != '{' { + continue + } + resp, err := ParseResponse(line) + if err == nil { + return resp, nil + } + return Response{}, exit(5, "parse flue protocol response: %v", err) + } + return Response{}, exit(5, "flue run stdout did not contain a protocol JSON response") +} diff --git a/internal/providers/flue/protocol_test.go b/internal/providers/flue/protocol_test.go new file mode 100644 index 000000000..5e87a963f --- /dev/null +++ b/internal/providers/flue/protocol_test.go @@ -0,0 +1,142 @@ +package flue + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestProtocolInputAndRequestRoundTrip(t *testing.T) { + input, err := ParseCLIInput([]byte(`{"requestFile":"/tmp/crabbox-flue-request.json"}`)) + if err != nil { + t.Fatal(err) + } + if input.RequestFile != "/tmp/crabbox-flue-request.json" { + t.Fatalf("requestFile=%q", input.RequestFile) + } + + req := validRequest() + data, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + parsed, err := ParseRequest(data) + if err != nil { + t.Fatal(err) + } + if parsed.ProtocolVersion != protocolVersion || parsed.Operation != operationRun || parsed.Command[0] != "go" { + t.Fatalf("parsed request=%#v", parsed) + } + limits := parsed.EffectiveOutputLimits() + if limits.StdoutBytes != defaultStdoutLimitBytes || limits.StderrBytes != defaultStderrLimitBytes { + t.Fatalf("limits=%#v", limits) + } +} + +func TestProtocolValidationRejectsMalformedRequests(t *testing.T) { + tests := []struct { + name string + edit func(*Request) + want string + }{ + {name: "version", edit: func(req *Request) { req.ProtocolVersion = 2 }, want: "protocolVersion"}, + {name: "operation", edit: func(req *Request) { req.Operation = "doctor" }, want: "operation"}, + {name: "workflow", edit: func(req *Request) { req.Workflow = "" }, want: "workflow"}, + {name: "target", edit: func(req *Request) { req.Target = "cloudflare" }, want: "unsupported"}, + {name: "archive", edit: func(req *Request) { req.WorkspaceArchive = "" }, want: "workspaceArchive"}, + {name: "workspace", edit: func(req *Request) { req.Workspace = "" }, want: "workspace"}, + {name: "command", edit: func(req *Request) { req.Command = nil }, want: "command"}, + {name: "timeout", edit: func(req *Request) { req.TimeoutMs = -1 }, want: "timeoutMs"}, + {name: "limits", edit: func(req *Request) { req.OutputLimits.StdoutBytes = -1 }, want: "output limits"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req := validRequest() + tc.edit(&req) + err := req.Validate() + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Validate err=%v want %q", err, tc.want) + } + }) + } +} + +func TestProtocolPreservesEmptyCommandArguments(t *testing.T) { + req := validRequest() + req.Command = []string{"printf", ""} + if err := req.Validate(); err != nil { + t.Fatalf("Validate rejected empty argv argument: %v", err) + } + req.Command = []string{""} + err := req.Validate() + if err == nil || !strings.Contains(err.Error(), "command[0]") { + t.Fatalf("Validate err=%v want command[0] rejection", err) + } +} + +func TestProtocolErrorsDoNotEchoSecretEnvValues(t *testing.T) { + req := validRequest() + req.Env = map[string]string{"TOKEN": "super-secret-token"} + req.OutputLimits.StdoutBytes = -1 + err := req.Validate() + if err == nil { + t.Fatal("Validate err=") + } + if strings.Contains(err.Error(), "super-secret-token") || strings.Contains(err.Error(), "TOKEN") { + t.Fatalf("validation error leaked env detail: %v", err) + } + + _, err = ParseRequest([]byte(`{"protocolVersion":1,"operation":"run","env":{"TOKEN":"super-secret-token"}`)) + if err == nil { + t.Fatal("ParseRequest err=") + } + if strings.Contains(err.Error(), "super-secret-token") || strings.Contains(err.Error(), "TOKEN") { + t.Fatalf("parse error leaked JSON detail: %v", err) + } +} + +func TestProtocolResponseValidation(t *testing.T) { + resp := Response{ + ProtocolVersion: protocolVersion, + Operation: operationRun, + ExitCode: 7, + Stdout: "ok", + Stderr: "failure", + Timing: ResponseTiming{TotalMs: 100, RunMs: 90}, + Artifacts: []ResponseArtifact{{Path: "logs/output.txt", SizeBytes: 12}}, + } + data, err := json.Marshal(resp) + if err != nil { + t.Fatal(err) + } + parsed, err := ParseResponse(data) + if err != nil { + t.Fatal(err) + } + if parsed.ExitCode != 7 || parsed.Stdout != "ok" || len(parsed.Artifacts) != 1 { + t.Fatalf("parsed response=%#v", parsed) + } + + resp.Artifacts[0].Path = "" + err = resp.Validate() + if err == nil || !strings.Contains(err.Error(), "artifact[0] path") { + t.Fatalf("Validate response err=%v", err) + } +} + +func validRequest() Request { + return Request{ + ProtocolVersion: protocolVersion, + Operation: operationRun, + LeaseID: "flue_123", + Slug: "sample", + Workflow: defaultWorkflow, + Target: defaultTarget, + WorkspaceArchive: "/tmp/workspace.tar.gz", + Workspace: defaultWorkdir, + Command: []string{"go", "test", "./..."}, + Env: map[string]string{"CI": "1"}, + TimeoutMs: 60000, + Metadata: map[string]string{"repo": "my-app"}, + } +} diff --git a/internal/providers/flue/provider.go b/internal/providers/flue/provider.go new file mode 100644 index 000000000..fd7e15799 --- /dev/null +++ b/internal/providers/flue/provider.go @@ -0,0 +1,225 @@ +package flue + +import ( + "context" + "flag" + "fmt" + "io" + "strings" + "time" + + core "github.com/openclaw/crabbox/internal/cli" +) + +func init() { + core.RegisterProvider(Provider{}) +} + +type Provider struct{} + +func (Provider) Name() string { return providerName } +func (Provider) Aliases() []string { return nil } + +func (Provider) Spec() core.ProviderSpec { + return core.ProviderSpec{ + Name: providerName, + Family: providerName, + Kind: core.ProviderKindDelegatedRun, + Targets: []core.TargetSpec{{OS: core.TargetLinux}}, + Features: core.FeatureSet{core.FeatureArchiveSync}, + Coordinator: core.CoordinatorNever, + } +} + +func (Provider) RegisterFlags(fs *flag.FlagSet, defaults core.Config) any { + return RegisterFlueProviderFlags(fs, defaults) +} + +func (Provider) ApplyFlags(cfg *core.Config, fs *flag.FlagSet, values any) error { + return ApplyFlueProviderFlags(cfg, fs, values) +} + +func (Provider) ValidateConfig(cfg core.Config) error { + return ValidateFlueConfig(cfg) +} + +func (Provider) ValidateDoctorConfig(cfg core.Config) error { + return ValidateFlueDoctorConfig(cfg) +} + +func (p Provider) Configure(cfg core.Config, rt core.Runtime) (core.Backend, error) { + if err := p.ValidateConfig(cfg); err != nil { + return nil, err + } + cfg.Provider = providerName + cfg.TargetOS = core.TargetLinux + return &backend{spec: p.Spec(), cfg: cfg, rt: rt}, nil +} + +func (p Provider) ConfigureDoctor(cfg core.Config, rt core.Runtime) (core.DoctorBackend, error) { + cfg.Provider = providerName + cfg.TargetOS = core.TargetLinux + return &backend{spec: p.Spec(), cfg: cfg, rt: rt}, nil +} + +type backend struct { + spec ProviderSpec + cfg Config + rt Runtime +} + +func (b *backend) Spec() ProviderSpec { return b.spec } + +func (b *backend) Warmup(context.Context, WarmupRequest) error { + return exit(2, "provider=%s is one-shot; use crabbox run", providerName) +} + +func (b *backend) Run(ctx context.Context, req RunRequest) (RunResult, error) { + if err := rejectFlueRunOptions(b.spec, b.cfg, req); err != nil { + return RunResult{}, err + } + started := b.now() + leaseID := newLeaseID() + slug := newLeaseSlug(leaseID) + if requested := strings.TrimSpace(req.RequestedSlug); requested != "" { + slug = normalizeLeaseSlug(requested) + } + commandText, err := flueCommandText(req.Command, req.ShellMode) + if err != nil { + return RunResult{}, err + } + cli, err := newFlueCLI(b.cfg, b.rt) + if err != nil { + return RunResult{}, err + } + payload, err := buildFlueRunPayload(ctx, b.cfg, req, leaseID, slug, started, b.rt.Stderr, b.now) + if err != nil { + return RunResult{}, err + } + defer payload.Cleanup() + if req.EnvSummary || len(req.Env) > 0 { + printEnvForwardingSummary(b.rt.Stderr, providerName, "forwarded in protocol request file", req.Options.EnvAllow, req.Env) + } + fmt.Fprintf(b.rt.Stderr, "provider=%s cli=%s target=node workflow=%s sync_delegated=true lifecycle=one-shot\n", providerName, blank(strings.TrimSpace(b.cfg.Flue.CLIPath), defaultCLIPath), workflowSelector(b.cfg.Flue.Workflow)) + commandStarted := b.now() + run, runErr := cli.run(ctx, payload.RequestFile, flueEnvRedactions(req.Env)) + commandDuration := b.now().Sub(commandStarted) + if run.Response.Timing.RunMs > 0 { + commandDuration = time.Duration(run.Response.Timing.RunMs) * time.Millisecond + } + exitCode := run.Response.ExitCode + result := RunResult{ + ExitCode: exitCode, + Command: commandDuration, + Total: b.now().Sub(started), + SyncDelegated: true, + Provider: providerName, + LeaseID: leaseID, + Slug: slug, + CommandText: commandText, + } + if runErr != nil { + if exitCode == 0 { + exitCode = flueProcessExitCode(run.Raw.ExitCode) + result.ExitCode = exitCode + } + result = finalizeRunResult(result, runErr) + _ = b.writeTiming(req, result, payload, commandDuration, runErr) + return result, runErr + } + if run.Response.Stdout != "" { + _, _ = io.WriteString(b.rt.Stdout, run.Response.Stdout) + } + if run.Response.Stderr != "" { + _, _ = io.WriteString(b.rt.Stderr, run.Response.Stderr) + } + var resultErr error + if strings.TrimSpace(run.Response.Error) != "" && exitCode == 0 { + resultErr = exit(1, "flue delegated run failed: %s", redactFlueDetail(run.Response.Error, flueEnvRedactions(req.Env))) + result.ExitCode = 1 + } else if exitCode != 0 { + message := fmt.Sprintf("flue delegated command exited %d", exitCode) + if strings.TrimSpace(run.Response.Error) != "" { + message += ": " + redactFlueDetail(run.Response.Error, flueEnvRedactions(req.Env)) + } + resultErr = exit(exitCode, "%s", message) + } + result = finalizeRunResult(result, resultErr) + if err := b.writeTiming(req, result, payload, commandDuration, resultErr); err != nil { + return result, err + } + fmt.Fprintf(b.rt.Stderr, "flue run summary sync=%s command=%s total=%s exit=%d\n", payload.SyncTotal.Round(time.Millisecond), commandDuration.Round(time.Millisecond), result.Total.Round(time.Millisecond), result.ExitCode) + if resultErr != nil { + return result, resultErr + } + return result, nil +} + +func (b *backend) List(context.Context, ListRequest) ([]LeaseView, error) { + return nil, nil +} + +func (b *backend) Status(context.Context, StatusRequest) (StatusView, error) { + return StatusView{}, exit(2, "provider=%s is one-shot and does not support status", providerName) +} + +func (b *backend) Stop(context.Context, StopRequest) error { + return exit(2, "provider=%s is one-shot and does not support stop", providerName) +} + +func (b *backend) now() time.Time { + if b.rt.Clock != nil { + return b.rt.Clock.Now() + } + return time.Now() +} + +func (b *backend) writeTiming(req RunRequest, result RunResult, payload flueRunPayload, commandDuration time.Duration, runErr error) error { + if !req.TimingJSON { + return nil + } + return writeTimingJSON(b.rt.Stderr, timingReportWithRunResult(timingReport{ + Provider: providerName, + LeaseID: result.LeaseID, + Slug: result.Slug, + SyncDelegated: true, + SyncMs: durationMillis(payload.SyncTotal), + SyncPhases: payload.SyncPhases, + CommandMs: durationMillis(commandDuration), + TotalMs: durationMillis(result.Total), + ExitCode: result.ExitCode, + Label: strings.TrimSpace(req.Label), + Workdir: payload.Request.Workspace, + }, result, runErr)) +} + +func rejectFlueRunOptions(spec ProviderSpec, cfg Config, req RunRequest) error { + if err := ValidateFlueConfig(cfg); err != nil { + return err + } + if err := rejectDelegatedSyncOptionsForSpec(spec, req); err != nil { + return err + } + if len(req.Command) == 0 { + return exit(2, "missing command") + } + if req.ID != "" || req.Keep || req.KeepOnFailure { + return exit(2, "provider=%s is one-shot and does not support persistent lease ids", providerName) + } + if req.NoSync { + return exit(2, "provider=%s requires archive sync; --no-sync is not supported", providerName) + } + if req.SyncOnly { + return exit(2, "provider=%s requires a Flue workflow command; --sync-only is not supported", providerName) + } + if req.Options.Desktop || req.Options.Browser || req.Options.Code { + return exit(2, "provider=%s does not support desktop, browser, or code-server options", providerName) + } + if req.Options.Tailscale.Enabled { + return exit(2, "provider=%s is delegated-run only and does not support Tailscale options", providerName) + } + if req.ApplyLocalPatch { + return exit(2, "provider=%s delegates sync; --apply-local-patch is not supported", providerName) + } + return nil +} diff --git a/internal/providers/flue/provider_test.go b/internal/providers/flue/provider_test.go new file mode 100644 index 000000000..a0852157a --- /dev/null +++ b/internal/providers/flue/provider_test.go @@ -0,0 +1,203 @@ +package flue + +import ( + "bytes" + "context" + "flag" + "strings" + "testing" + + core "github.com/openclaw/crabbox/internal/cli" +) + +func TestProviderSpecMatchesContract(t *testing.T) { + p := Provider{} + if p.Name() != providerName { + t.Fatalf("Name=%q want %q", p.Name(), providerName) + } + if p.Aliases() != nil { + t.Fatalf("Aliases=%v, want nil", p.Aliases()) + } + spec := p.Spec() + if spec.Name != providerName || spec.Family != providerName { + t.Fatalf("spec identity=%#v", spec) + } + if spec.Kind != core.ProviderKindDelegatedRun { + t.Fatalf("Kind=%q want delegated-run", spec.Kind) + } + if len(spec.Targets) != 1 || spec.Targets[0].OS != core.TargetLinux { + t.Fatalf("Targets=%#v, want Linux only", spec.Targets) + } + if !spec.Features.Has(core.FeatureArchiveSync) || len(spec.Features) != 1 { + t.Fatalf("Features=%#v, want archive-sync only", spec.Features) + } + if spec.Coordinator != core.CoordinatorNever { + t.Fatalf("Coordinator=%q want never", spec.Coordinator) + } +} + +func TestApplyFlagsAndValidate(t *testing.T) { + cfg := core.BaseConfig() + cfg.Provider = providerName + fs := flag.NewFlagSet("test", flag.ContinueOnError) + values := Provider{}.RegisterFlags(fs, cfg) + args := []string{ + "--flue-cli", "/opt/flue/bin/flue", + "--flue-root", "/repo/flue", + "--flue-workflow", "workflow:test", + "--flue-target", "node", + "--flue-config", "/repo/flue/flue.config.ts", + "--flue-env", "/repo/flue/.env", + "--flue-output", "json", + "--flue-workdir", "/workspace/app", + "--flue-timeout-secs", "123", + } + if err := fs.Parse(args); err != nil { + t.Fatal(err) + } + if err := (Provider{}).ApplyFlags(&cfg, fs, values); err != nil { + t.Fatal(err) + } + want := core.FlueConfig{ + CLIPath: "/opt/flue/bin/flue", + Root: "/repo/flue", + Workflow: "workflow:test", + Target: "node", + Config: "/repo/flue/flue.config.ts", + EnvFile: "/repo/flue/.env", + Output: "json", + Workdir: "/workspace/app", + TimeoutSecs: 123, + } + if cfg.Flue != want { + t.Fatalf("cfg.Flue=%#v want %#v", cfg.Flue, want) + } +} + +func TestApplyFlagsRejectsUnsupportedGenericSizing(t *testing.T) { + for _, flagName := range []string{"class", "type"} { + cfg := core.BaseConfig() + cfg.Provider = providerName + fs := flag.NewFlagSet("test", flag.ContinueOnError) + values := Provider{}.RegisterFlags(fs, cfg) + fs.String(flagName, "", "generic") + if err := fs.Parse([]string{"--" + flagName, "x"}); err != nil { + t.Fatal(err) + } + err := Provider{}.ApplyFlags(&cfg, fs, values) + if err == nil || !strings.Contains(err.Error(), "--"+flagName+" is not supported") { + t.Fatalf("ApplyFlags with --%s err=%v", flagName, err) + } + } +} + +func TestApplyFlagsAllowsUnsupportedTargetForDoctorDiagnostics(t *testing.T) { + cfg := core.BaseConfig() + cfg.Provider = providerName + fs := flag.NewFlagSet("test", flag.ContinueOnError) + values := Provider{}.RegisterFlags(fs, cfg) + if err := fs.Parse([]string{"--flue-target", "cloudflare"}); err != nil { + t.Fatal(err) + } + if err := (Provider{}).ApplyFlags(&cfg, fs, values); err != nil { + t.Fatalf("ApplyFlags err=%v", err) + } + if cfg.Flue.Target != "cloudflare" { + t.Fatalf("target=%q want cloudflare", cfg.Flue.Target) + } +} + +func TestValidateFlueConfigRejectsUnsupportedValues(t *testing.T) { + tests := []struct { + name string + edit func(*core.Config) + want string + }{ + { + name: "empty cli", + edit: func(cfg *core.Config) { cfg.Flue.CLIPath = "" }, + want: "cliPath", + }, + { + name: "empty workflow", + edit: func(cfg *core.Config) { cfg.Flue.Workflow = "" }, + want: "workflow", + }, + { + name: "cloudflare target", + edit: func(cfg *core.Config) { cfg.Flue.Target = "cloudflare" }, + want: "target=node only", + }, + { + name: "server target", + edit: func(cfg *core.Config) { cfg.Flue.Target = "server" }, + want: "target=node only", + }, + { + name: "relative workdir", + edit: func(cfg *core.Config) { cfg.Flue.Workdir = "workspace" }, + want: "must be absolute", + }, + { + name: "broad workdir", + edit: func(cfg *core.Config) { cfg.Flue.Workdir = "/workspace" }, + want: "too broad", + }, + { + name: "negative timeout", + edit: func(cfg *core.Config) { cfg.Flue.TimeoutSecs = -1 }, + want: "non-negative", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := core.BaseConfig() + cfg.Provider = providerName + tc.edit(&cfg) + err := ValidateFlueConfig(cfg) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("ValidateFlueConfig err=%v want %q", err, tc.want) + } + }) + } +} + +func TestValidateFlueRunTargetRejectsUnsupportedValues(t *testing.T) { + for _, target := range []string{"cloudflare", "server"} { + cfg := core.BaseConfig() + cfg.Provider = providerName + cfg.Flue.Target = target + err := ValidateFlueRunTarget(cfg) + if err == nil || !strings.Contains(err.Error(), "target=node only") { + t.Fatalf("ValidateFlueRunTarget(%q) err=%v", target, err) + } + } +} + +func TestConfigureReturnsDelegatedBackend(t *testing.T) { + cfg := core.BaseConfig() + cfg.Provider = providerName + var out, stderr bytes.Buffer + backend, err := Provider{}.Configure(cfg, core.Runtime{Stdout: &out, Stderr: &stderr}) + if err != nil { + t.Fatal(err) + } + delegated, ok := backend.(core.DelegatedRunBackend) + if !ok { + t.Fatalf("backend does not implement DelegatedRunBackend: %T", backend) + } + _, err = delegated.Run(context.Background(), core.RunRequest{}) + if err == nil || !strings.Contains(err.Error(), "missing command") { + t.Fatalf("Run err=%v", err) + } +} + +func TestConfigureRejectsUnsupportedTarget(t *testing.T) { + cfg := core.BaseConfig() + cfg.Provider = providerName + cfg.Flue.Target = "cloudflare" + _, err := Provider{}.Configure(cfg, core.Runtime{}) + if err == nil || !strings.Contains(err.Error(), "target=node only") { + t.Fatalf("Configure err=%v", err) + } +} diff --git a/internal/providers/flue/sync.go b/internal/providers/flue/sync.go new file mode 100644 index 000000000..1e1ad435c --- /dev/null +++ b/internal/providers/flue/sync.go @@ -0,0 +1,194 @@ +package flue + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "time" +) + +type flueRunPayload struct { + RequestFile string + ArchiveFile *os.File + Request Request + SyncPhases []timingPhase + SyncTotal time.Duration + Cleanup func() +} + +func buildFlueRunPayload(ctx context.Context, cfg Config, req RunRequest, leaseID, slug string, started time.Time, stderr io.Writer, now func() time.Time) (flueRunPayload, error) { + if strings.TrimSpace(req.Repo.Root) == "" { + return flueRunPayload{}, exit(2, "provider=%s requires a local git workspace for archive sync", providerName) + } + workdir, err := cleanWorkdir(cfg.Flue.Workdir) + if err != nil { + return flueRunPayload{}, err + } + command, err := flueCommand(req.Command, req.ShellMode) + if err != nil { + return flueRunPayload{}, err + } + syncStart := now() + excludes, err := syncExcludes(req.Repo.Root, cfg) + if err != nil { + return flueRunPayload{}, err + } + manifestStart := now() + manifest, err := syncManifest(req.Repo.Root, excludes, cfg.Sync.Includes) + if err != nil { + return flueRunPayload{}, exit(6, "build sync file list: %v", err) + } + manifestDuration := now().Sub(manifestStart) + preflightStart := now() + archiveManifest := manifest + archiveManifest.Changed = nil + archiveManifest.ChangedBytes = 0 + if stderr == nil { + stderr = io.Discard + } + if err := checkSyncPreflight(archiveManifest, cfg, req.ForceSyncLarge, stderr); err != nil { + return flueRunPayload{}, err + } + preflightDuration := now().Sub(preflightStart) + archiveStart := now() + archive, err := createPortableSyncArchive(ctx, req.Repo, manifest, "crabbox-flue-sync-*.tgz") + if err != nil { + return flueRunPayload{}, err + } + cleanupArchive := true + cleanup := func() { + if archive != nil { + name := archive.Name() + _ = archive.Close() + if cleanupArchive { + _ = os.Remove(name) + } + } + } + archiveDuration := now().Sub(archiveStart) + request := Request{ + ProtocolVersion: protocolVersion, + Operation: operationRun, + LeaseID: leaseID, + Slug: slug, + Workflow: strings.TrimSpace(cfg.Flue.Workflow), + Target: strings.ToLower(blank(strings.TrimSpace(cfg.Flue.Target), defaultTarget)), + WorkspaceArchive: archive.Name(), + Workspace: workdir, + Command: command, + Env: req.Env, + TimeoutMs: int64(cfg.Flue.TimeoutSecs) * int64(time.Second/time.Millisecond), + Metadata: map[string]string{ + "provider": providerName, + "repo": strings.TrimSpace(req.Repo.Name), + "started": started.UTC().Format(time.RFC3339Nano), + }, + OutputLimits: OutputLimits{ + StdoutBytes: defaultStdoutLimitBytes, + StderrBytes: defaultStderrLimitBytes, + }, + } + if request.TimeoutMs == 0 { + request.TimeoutMs = 0 + } + if err := request.Validate(); err != nil { + cleanup() + return flueRunPayload{}, err + } + requestFile, err := writeFlueRequestFile(request) + if err != nil { + cleanup() + return flueRunPayload{}, err + } + cleanupRequest := true + cleanupAll := func() { + if cleanupRequest { + _ = os.Remove(requestFile) + } + cleanup() + } + syncTotal := now().Sub(syncStart) + return flueRunPayload{ + RequestFile: requestFile, + ArchiveFile: archive, + Request: request, + SyncPhases: []timingPhase{ + {Name: "manifest", Ms: manifestDuration.Milliseconds()}, + {Name: "preflight", Ms: preflightDuration.Milliseconds()}, + {Name: "archive", Ms: archiveDuration.Milliseconds()}, + {Name: "flue_archive_sync", Ms: syncTotal.Milliseconds()}, + }, + SyncTotal: syncTotal, + Cleanup: cleanupAll, + }, nil +} + +func writeFlueRequestFile(request Request) (string, error) { + file, err := os.CreateTemp("", "crabbox-flue-request-*.json") + if err != nil { + return "", fmt.Errorf("create flue request temp file: %w", err) + } + path := file.Name() + keep := false + defer func() { + if !keep { + _ = file.Close() + _ = os.Remove(path) + } + }() + if err := file.Chmod(0o600); err != nil { + return "", fmt.Errorf("secure flue request temp file: %w", err) + } + encoder := json.NewEncoder(file) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(request); err != nil { + return "", fmt.Errorf("write flue request temp file: %w", err) + } + if err := file.Close(); err != nil { + return "", fmt.Errorf("close flue request temp file: %w", err) + } + keep = true + return path, nil +} + +func flueCommand(command []string, shellMode bool) ([]string, error) { + if len(command) == 0 { + return nil, exit(2, "missing command") + } + if shellMode { + return []string{"/bin/sh", "-lc", strings.Join(command, " ")}, nil + } + if len(command) == 1 && shouldUseShell(command) { + return []string{"/bin/sh", "-lc", command[0]}, nil + } + if shouldUseShell(command) || leadingEnvAssignment(command) { + return []string{"/bin/sh", "-lc", shellScriptFromArgv(command)}, nil + } + return append([]string(nil), command...), nil +} + +func flueCommandText(command []string, shellMode bool) (string, error) { + if len(command) == 0 { + return "", exit(2, "missing command") + } + if shellMode { + return strings.Join(command, " "), nil + } + if len(command) == 1 && shouldUseShell(command) { + return command[0], nil + } + return shellScriptFromArgv(command), nil +} + +func flueEnvRedactions(env map[string]string) []string { + values := make([]string, 0, len(env)) + for _, value := range env { + if strings.TrimSpace(value) != "" { + values = append(values, value) + } + } + return values +}