diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 00000000..e6c80ab4 --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1,8 @@ +# Benchmark results (generated per-run, not tracked) +results/ + +# Temporary scoring files +_bench_harness.forge.js +_bench_candidate.forge.js +_metrics_harness.forge.js +solution.forge.js diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..48a80fb4 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,92 @@ +# ForgeCAD Benchmark — Vision-Based 3D Reconstruction + +An AI benchmark that measures how well language models can reproduce 3D geometry from reference images. + +## How it works + +1. **Reference model** — A `.forge.js` file defines the ground-truth 3D geometry +2. **Preparation** — Multi-angle images are rendered and reference metrics are computed +3. **Agentic loop** — The AI sees the reference images, writes ForgeCAD code, receives spatial scores and comparison renders, and iterates +4. **Scoring** — The candidate model is compared to the reference using: + - **3D IoU** (50%) — volumetric intersection-over-union (the gold standard) + - **Volume match** (20%) — how close the total volume is + - **Bounding box match** (15%) — how close the overall dimensions are + - **Surface area match** (15%) — how close the surface complexity is + +## Quick start + +```bash +# 1. Prepare a challenge from any .forge.js model +node bench/prepare.mjs examples/cup.forge.js cup --difficulty easy + +# 2. Score a single solution +node bench/score.mjs bench/challenges/cup my-solution.forge.js + +# 3. Run the full agentic loop with an LLM +ANTHROPIC_API_KEY=... node bench/runner.mjs \ + --challenge bench/challenges/cup \ + --model claude-sonnet-4-20250514 \ + --iterations 5 + +# 4. Run all challenges +ANTHROPIC_API_KEY=... node bench/runner.mjs --all --model claude-sonnet-4-20250514 +``` + +## Directory structure + +``` +bench/ + prepare.mjs # Prepare a challenge from a reference model + score.mjs # Score a candidate solution against reference + runner.mjs # Agentic LLM loop with vision + api-reference.md # ForgeCAD API docs given to the LLM + challenges/ # Benchmark challenges + / + reference.forge.js # Ground-truth model + config.json # Challenge metadata + prepared/ + metrics.json # Reference volume, bbox, surface area + views/ # Multi-angle reference renders (front, right, top, iso) + results/ # Run results (gitignored) + / + summary.json + best-solution.forge.js + iterations/ + 0/ + code.forge.js + score.json + llm-response.txt + views/ # Candidate renders for comparison +``` + +## Validators + +The scoring system uses spatial validators that compare the candidate mesh to the reference: + +| Validator | Weight | What it measures | +|-----------|--------|------------------| +| 3D IoU | 50% | Volumetric overlap after centering both shapes | +| Volume | 20% | Ratio of smaller/larger volume | +| Bounding box | 15% | Dimension similarity (orientation-independent) | +| Surface area | 15% | Surface complexity similarity | + +The 3D IoU is the most meaningful metric — it directly measures how much the two shapes overlap in 3D space. A perfect cube will score poorly against a perfect sphere even if they have similar volumes. + +Future validators will include functional tests (e.g., does this bolt actually fit through this hole). + +## Adding challenges + +```bash +# From an existing example +node bench/prepare.mjs examples/bottle.forge.js bottle --difficulty hard + +# From a custom model +node bench/prepare.mjs path/to/my-model.forge.js my-challenge --difficulty medium +``` + +The `prepare` step renders reference images (requires Chrome/Puppeteer). If rendering isn't available, you can add images manually to `bench/challenges//prepared/views/`. + +## Providers + +- `--provider anthropic` (default) — uses `ANTHROPIC_API_KEY` +- `--provider openrouter` — uses `OPENROUTER_API_KEY`, supports any model via OpenRouter diff --git a/bench/api-reference.md b/bench/api-reference.md new file mode 100644 index 00000000..9d0ced7a --- /dev/null +++ b/bench/api-reference.md @@ -0,0 +1,96 @@ +# ForgeCAD API Reference (Benchmark Subset) + +ForgeCAD scripts are JavaScript. The API is globally available — no imports needed. +Scripts must `return` geometry (Shape, Assembly, or SolvedAssembly). + +## Primitives + +```javascript +box(x, y, z, center?) // Rectangular box. center=true → centered at origin +cylinder(height, radius, radiusTop?, segments?, center?) // Cylinder/cone +sphere(radius, segments?) // Sphere at origin +``` + +## Booleans + +```javascript +union(a, b, ...) // Combine shapes +difference(a, b, ...) // Subtract b,c,... from a +intersection(a, b, ...) // Keep overlap only +shape.subtract(other) // Shorthand for difference +``` + +## Transforms + +```javascript +shape.translate(x, y, z) // Move +shape.rotate(rx, ry, rz) // Rotate (degrees, Euler XYZ) +shape.scale(s) // Uniform scale +shape.mirror([nx, ny, nz]) // Mirror across plane through origin +shape.color("#hex") // Set color +``` + +## Measurements + +```javascript +shape.boundingBox() // { min: [x,y,z], max: [x,y,z] } +shape.volume() // mm³ +shape.surfaceArea() // mm² +shape.isEmpty() // boolean +``` + +## Assembly + +```javascript +assembly(name) // Create assembly + .addPart(name, shape, options?) // Add a named part + .addRevolute(name, parent, child, opts) // Revolute (hinge) joint + .addPrismatic(name, parent, child, opts) // Prismatic (slide) joint + .addFixed(name, parent, child, opts) // Fixed connection + .solve(state?) // Solve at joint values → SolvedAssembly + .sweepJoint(name, from, to, steps, base?) // Sweep joint, check collisions + .describe() // Get parts/joints metadata + +// Joint options: +// { axis: [x,y,z], min: deg, max: deg, default: deg, +// frame: Transform.identity().translate(x,y,z) } + +// SolvedAssembly: +solved.getPart(name) // Get positioned part +solved.collisionReport() // Check for part overlaps +solved.minClearance(a, b, len?) // Min gap between two parts +solved.toGroup() // Convert to ShapeGroup for display +``` + +## Gear Coupling + +```javascript +const pair = lib.gearPair({ + pinion: { module: m, teeth: n, faceWidth: w }, + gear: { module: m, teeth: n, faceWidth: w }, +}); + +assembly.addGearCoupling(drivenJoint, driverJoint, { pair }) +// Auto-computes ratio from tooth counts +``` + +## Transforms for Joint Frames + +```javascript +Transform.identity() // Identity transform +Transform.identity().translate(x, y, z) // Translation +``` + +## Verification (optional, for self-checks) + +```javascript +verify.that(label, () => condition, message?) +verify.equal(label, actual, expected, tolerance?) +verify.inRange(label, value, min, max) +verify.notColliding(label, shapeA, shapeB) +``` + +## Coordinate System + +- **Z-up** right-handed: X = left/right, Y = forward/back, Z = up/down +- All dimensions in millimeters, angles in degrees diff --git a/bench/challenges/bracket/config.json b/bench/challenges/bracket/config.json new file mode 100644 index 00000000..fc11bdfd --- /dev/null +++ b/bench/challenges/bracket/config.json @@ -0,0 +1,36 @@ +{ + "name": "bracket", + "description": "Reproduce the bracket model from reference images", + "difficulty": "easy", + "referenceModel": "reference.forge.js", + "preparedAt": "2026-03-30T10:13:24.811Z", + "metrics": { + "volume": 10834.77, + "surfaceArea": 5452, + "boundingBox": { + "min": [ + 0, + 0, + 0 + ], + "max": [ + 40, + 30, + 30 + ], + "size": [ + 40, + 30, + 30 + ] + }, + "isEmpty": false + }, + "imageSize": 512, + "validators": [ + "iou", + "volume", + "bbox", + "surfaceArea" + ] +} \ No newline at end of file diff --git a/bench/challenges/bracket/prepared/metrics.json b/bench/challenges/bracket/prepared/metrics.json new file mode 100644 index 00000000..c08c2431 --- /dev/null +++ b/bench/challenges/bracket/prepared/metrics.json @@ -0,0 +1,22 @@ +{ + "volume": 10834.77, + "surfaceArea": 5452, + "boundingBox": { + "min": [ + 0, + 0, + 0 + ], + "max": [ + 40, + 30, + 30 + ], + "size": [ + 40, + 30, + 30 + ] + }, + "isEmpty": false +} \ No newline at end of file diff --git a/bench/challenges/bracket/reference.forge.js b/bench/challenges/bracket/reference.forge.js new file mode 100644 index 00000000..d4d7c8c9 --- /dev/null +++ b/bench/challenges/bracket/reference.forge.js @@ -0,0 +1,25 @@ +// Reference: L-bracket with two bolt holes +const plateW = 40; +const plateH = 30; +const thickness = 5; +const holeD = 6.5; // M6 clearance + +// Vertical plate (XZ plane) +const vertical = box(plateW, thickness, plateH); + +// Horizontal plate (XY plane) +const horizontal = box(plateW, plateH, thickness); + +// Join into L-shape +const lShape = union(vertical, horizontal); + +// Hole in vertical plate (through Y axis) +const hole1 = cylinder(thickness + 2, holeD / 2) + .rotate(90, 0, 0) + .translate(plateW / 2, thickness / 2, plateH / 2); + +// Hole in horizontal plate (through Z axis) +const hole2 = cylinder(thickness + 2, holeD / 2) + .translate(plateW / 2, plateH / 2, thickness / 2); + +return lShape.subtract(hole1).subtract(hole2); diff --git a/bench/challenges/cup/config.json b/bench/challenges/cup/config.json new file mode 100644 index 00000000..c62bcb6c --- /dev/null +++ b/bench/challenges/cup/config.json @@ -0,0 +1,36 @@ +{ + "name": "cup", + "description": "Reproduce the cup model from reference images", + "difficulty": "easy", + "referenceModel": "reference.forge.js", + "preparedAt": "2026-03-30T10:13:13.773Z", + "metrics": { + "volume": 54547.91, + "surfaceArea": 32146.2, + "boundingBox": { + "min": [ + -35, + -35, + 0 + ], + "max": [ + 35, + 35, + 80 + ], + "size": [ + 70, + 70, + 80 + ] + }, + "isEmpty": false + }, + "imageSize": 512, + "validators": [ + "iou", + "volume", + "bbox", + "surfaceArea" + ] +} \ No newline at end of file diff --git a/bench/challenges/cup/prepared/metrics.json b/bench/challenges/cup/prepared/metrics.json new file mode 100644 index 00000000..2022e1a7 --- /dev/null +++ b/bench/challenges/cup/prepared/metrics.json @@ -0,0 +1,22 @@ +{ + "volume": 54547.91, + "surfaceArea": 32146.2, + "boundingBox": { + "min": [ + -35, + -35, + 0 + ], + "max": [ + 35, + 35, + 80 + ], + "size": [ + 70, + 70, + 80 + ] + }, + "isEmpty": false +} \ No newline at end of file diff --git a/bench/challenges/cup/reference.forge.js b/bench/challenges/cup/reference.forge.js new file mode 100644 index 00000000..7b6762b9 --- /dev/null +++ b/bench/challenges/cup/reference.forge.js @@ -0,0 +1,12 @@ +// Reference: Simple tapered cup with hollow interior +const height = 80; +const topRadius = 35; +const bottomRadius = 25; +const wallThickness = 3; +const baseThickness = 5; + +const outer = cylinder(height, bottomRadius, topRadius); +const inner = cylinder(height - baseThickness, bottomRadius - wallThickness, topRadius - wallThickness) + .translate(0, 0, baseThickness); + +return outer.subtract(inner); diff --git a/bench/challenges/pipe-tee/config.json b/bench/challenges/pipe-tee/config.json new file mode 100644 index 00000000..4b1ac272 --- /dev/null +++ b/bench/challenges/pipe-tee/config.json @@ -0,0 +1,36 @@ +{ + "name": "pipe-tee", + "description": "Reproduce the pipe-tee model from reference images", + "difficulty": "medium", + "referenceModel": "reference.forge.js", + "preparedAt": "2026-03-30T10:13:30.458Z", + "metrics": { + "volume": 28338.9, + "surfaceArea": 19643.93, + "boundingBox": { + "min": [ + 25, + -15, + -15 + ], + "max": [ + 120, + 15, + 40 + ], + "size": [ + 95, + 30, + 55 + ] + }, + "isEmpty": false + }, + "imageSize": 512, + "validators": [ + "iou", + "volume", + "bbox", + "surfaceArea" + ] +} \ No newline at end of file diff --git a/bench/challenges/pipe-tee/prepared/metrics.json b/bench/challenges/pipe-tee/prepared/metrics.json new file mode 100644 index 00000000..83a7de43 --- /dev/null +++ b/bench/challenges/pipe-tee/prepared/metrics.json @@ -0,0 +1,22 @@ +{ + "volume": 28338.9, + "surfaceArea": 19643.93, + "boundingBox": { + "min": [ + 25, + -15, + -15 + ], + "max": [ + 120, + 15, + 40 + ], + "size": [ + 95, + 30, + 55 + ] + }, + "isEmpty": false +} \ No newline at end of file diff --git a/bench/challenges/pipe-tee/reference.forge.js b/bench/challenges/pipe-tee/reference.forge.js new file mode 100644 index 00000000..6956119b --- /dev/null +++ b/bench/challenges/pipe-tee/reference.forge.js @@ -0,0 +1,19 @@ +// Reference: T-junction pipe fitting +const outerR = 15; +const innerR = 12; +const mainLen = 80; +const branchLen = 40; + +// Main pipe along X axis +const mainOuter = cylinder(mainLen, outerR).rotate(0, 90, 0).translate(mainLen / 2, 0, 0); +const mainInner = cylinder(mainLen + 2, innerR).rotate(0, 90, 0).translate(mainLen / 2, 0, 0); + +// Branch pipe along Z axis, rising from the center of the main pipe +const branchOuter = cylinder(branchLen, outerR).translate(mainLen / 2, 0, 0); +const branchInner = cylinder(branchLen + outerR, innerR).translate(mainLen / 2, 0, 0); + +// Union outer shells, then subtract inner bores +const outer = union(mainOuter, branchOuter); +const inner = union(mainInner, branchInner); + +return outer.subtract(inner); diff --git a/bench/prepare.mjs b/bench/prepare.mjs new file mode 100644 index 00000000..67f1dac7 --- /dev/null +++ b/bench/prepare.mjs @@ -0,0 +1,192 @@ +#!/usr/bin/env node +/** + * ForgeCAD Benchmark — Challenge Preparation + * + * Takes a reference .forge.js model and prepares a benchmark challenge: + * 1. Validates the reference model executes correctly + * 2. Renders multi-angle reference images (front, right, top, iso) + * 3. Computes and stores reference metrics (volume, bbox, surface area) + * + * Usage: + * node bench/prepare.mjs + * node bench/prepare.mjs examples/cup.forge.js cup + * + * Output structure: + * bench/challenges// + * reference.forge.js + * config.json + * prepared/ + * metrics.json + * views/ + * reference_front.png + * reference_right.png + * reference_top.png + * reference_iso.png + */ + +import { readFileSync, writeFileSync, existsSync, mkdirSync, copyFileSync, unlinkSync } from 'fs'; +import { execSync } from 'child_process'; +import { join, resolve, dirname, basename } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PROJECT_ROOT = resolve(__dirname, '..'); +const FORGECAD_CLI = join(PROJECT_ROOT, 'dist-cli', 'forgecad.js'); +const CHALLENGES_DIR = join(__dirname, 'challenges'); + +// --------------------------------------------------------------------------- +// Metrics extraction harness — runs reference model and outputs JSON metrics +// --------------------------------------------------------------------------- + +const METRICS_HARNESS = ` +// Auto-generated metrics extraction harness +const shape = require("./reference.forge.js"); +const vol = shape.volume(); +const sa = shape.surfaceArea(); +const bb = shape.boundingBox(); +const size = [bb.max[0]-bb.min[0], bb.max[1]-bb.min[1], bb.max[2]-bb.min[2]]; +const metrics = { + volume: Math.round(vol * 100) / 100, + surfaceArea: Math.round(sa * 100) / 100, + boundingBox: { + min: bb.min.map(v => Math.round(v * 100) / 100), + max: bb.max.map(v => Math.round(v * 100) / 100), + size: size.map(v => Math.round(v * 100) / 100), + }, + isEmpty: shape.isEmpty(), +}; +console.warn("BENCH_METRICS:" + JSON.stringify(metrics)); +return shape; +`; + +// --------------------------------------------------------------------------- +// Prepare a challenge +// --------------------------------------------------------------------------- + +/** + * @param {string} referenceModelPath Path to the reference .forge.js file + * @param {string} challengeName Name for the challenge directory + * @param {object} [options] + * @param {number} [options.imageSize] Render size in px (default 512) + * @param {string} [options.difficulty] Difficulty label + * @param {string} [options.description] Challenge description + */ +export function prepareChallenge(referenceModelPath, challengeName, options = {}) { + const absRef = resolve(referenceModelPath); + if (!existsSync(absRef)) { + throw new Error(`Reference model not found: ${absRef}`); + } + + const challengeDir = join(CHALLENGES_DIR, challengeName); + const preparedDir = join(challengeDir, 'prepared'); + const viewsDir = join(preparedDir, 'views'); + + // Create directories + mkdirSync(viewsDir, { recursive: true }); + + // Copy reference model + const refDest = join(challengeDir, 'reference.forge.js'); + copyFileSync(absRef, refDest); + console.log(` Copied reference model → ${refDest}`); + + // --- Step 1: Extract metrics --- + console.log(' Extracting reference metrics...'); + const harnessFile = join(challengeDir, '_metrics_harness.forge.js'); + writeFileSync(harnessFile, METRICS_HARNESS, 'utf8'); + + let metrics; + try { + const output = execSync( + `node "${FORGECAD_CLI}" run "${harnessFile}" 2>&1`, + { encoding: 'utf8', timeout: 60_000, cwd: PROJECT_ROOT }, + ); + const match = output.match(/BENCH_METRICS:(.+)/); + if (!match) throw new Error('No BENCH_METRICS in output:\n' + output); + metrics = JSON.parse(match[1]); + } finally { + try { unlinkSync(harnessFile); } catch {} + } + + if (metrics.isEmpty) { + throw new Error('Reference model produced an empty shape'); + } + + writeFileSync(join(preparedDir, 'metrics.json'), JSON.stringify(metrics, null, 2), 'utf8'); + console.log(` Metrics: vol=${metrics.volume}mm³ SA=${metrics.surfaceArea}mm² bbox=${metrics.boundingBox.size.join('×')}mm`); + + // --- Step 2: Render multi-angle images --- + console.log(' Rendering reference views...'); + const imageSize = options.imageSize || 512; + const outputBase = join(viewsDir, 'reference.png'); + + try { + execSync( + `node "${FORGECAD_CLI}" render "${refDest}" "${outputBase}" --angles front,right,top,iso --size ${imageSize} 2>&1`, + { encoding: 'utf8', timeout: 120_000, cwd: PROJECT_ROOT }, + ); + console.log(' Rendered: front, right, top, iso'); + } catch (e) { + console.warn(` Warning: Rendering failed (Puppeteer/Chrome may not be available)`); + console.warn(` ${e.message.split('\n')[0]}`); + console.warn(' You can render manually: forgecad render --angles front,right,top,iso'); + } + + // --- Step 3: Write config --- + const config = { + name: challengeName, + description: options.description || `Reproduce the ${challengeName} model from reference images`, + difficulty: options.difficulty || 'medium', + referenceModel: 'reference.forge.js', + preparedAt: new Date().toISOString(), + metrics, + imageSize, + validators: ['iou', 'volume', 'bbox', 'surfaceArea'], + }; + writeFileSync(join(challengeDir, 'config.json'), JSON.stringify(config, null, 2), 'utf8'); + + console.log(` Challenge prepared → ${challengeDir}`); + return { challengeDir, config, metrics }; +} + +// --------------------------------------------------------------------------- +// CLI entry point +// --------------------------------------------------------------------------- + +const isMainModule = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMainModule) { + const args = process.argv.slice(2); + const positional = args.filter(a => !a.startsWith('--')); + + if (positional.length < 2) { + console.error('Usage: node bench/prepare.mjs [options]'); + console.error(''); + console.error('Options:'); + console.error(' --size Image render size (default 512)'); + console.error(' --difficulty easy | medium | hard'); + console.error(' --description Challenge description'); + console.error(''); + console.error('Example:'); + console.error(' node bench/prepare.mjs examples/cup.forge.js cup --difficulty easy'); + process.exit(1); + } + + const [referenceModel, challengeName] = positional; + + const getArg = (name) => { + const idx = args.indexOf(`--${name}`); + return idx !== -1 ? args[idx + 1] : undefined; + }; + + console.log(`\nPreparing challenge "${challengeName}" from ${referenceModel}\n`); + try { + prepareChallenge(referenceModel, challengeName, { + imageSize: parseInt(getArg('size')) || undefined, + difficulty: getArg('difficulty'), + description: getArg('description'), + }); + console.log('\nDone.\n'); + } catch (e) { + console.error(`\nFatal: ${e.message}\n`); + process.exit(1); + } +} diff --git a/bench/runner.js b/bench/runner.js deleted file mode 100644 index c182657b..00000000 --- a/bench/runner.js +++ /dev/null @@ -1,319 +0,0 @@ -#!/usr/bin/env node -/** - * ForgeCAD Bench Runner - * - * Runs a functional test harness against a solution — either a local file - * or one generated by an LLM via OpenRouter. - * - * Usage: - * # Test a local solution file - * node bench/runner.js --task examples/bench/simple-tongs --solution path/to/solution.forge.js - * - * # Generate + test via LLM (OpenRouter) - * node bench/runner.js --task examples/bench/simple-tongs --model minimax/minimax-m2.7 - * - * # Run all tasks against an LLM - * node bench/runner.js --all --model minimax/minimax-m2.7 - * - * Environment: - * OPENROUTER_API_KEY — required for --model mode - */ - -import { readFileSync, writeFileSync, copyFileSync, unlinkSync, existsSync, readdirSync } from "fs"; -import { execSync } from "child_process"; -import { join, resolve, basename } from "path"; - -// --------------------------------------------------------------------------- -// CLI argument parsing -// --------------------------------------------------------------------------- - -const args = process.argv.slice(2); -function getArg(name) { - const idx = args.indexOf(`--${name}`); - if (idx === -1) return null; - return args[idx + 1] || null; -} -const hasFlag = (name) => args.includes(`--${name}`); - -const taskDir = getArg("task"); -const solutionPath = getArg("solution"); -const modelId = getArg("model"); -const runAll = hasFlag("all"); -const parallel = hasFlag("parallel"); -const verbose = hasFlag("verbose"); -const apiKey = process.env.OPENROUTER_API_KEY || getArg("api-key"); - -if (!runAll && !taskDir) { - console.error("Usage: node bench/runner.js --task [--solution | --model ]"); - console.error(" node bench/runner.js --all --model "); - process.exit(1); -} - -if (modelId && !apiKey) { - console.error("Error: OPENROUTER_API_KEY env var or --api-key required for LLM mode"); - process.exit(1); -} - -// --------------------------------------------------------------------------- -// Discover tasks -// --------------------------------------------------------------------------- - -const benchRoot = resolve(getArg("bench-dir") || "examples/bench-v2"); - -function discoverTasks() { - const dirs = readdirSync(benchRoot, { withFileTypes: true }) - .filter((d) => d.isDirectory()) - .map((d) => join(benchRoot, d.name)) - .filter((d) => existsSync(join(d, "harness.forge.js")) && existsSync(join(d, "prompt.md"))); - return dirs; -} - -// --------------------------------------------------------------------------- -// LLM generation via OpenRouter -// --------------------------------------------------------------------------- - -async function generateSolution(task, model) { - const prompt = readFileSync(join(task, "prompt.md"), "utf8"); - const apiRef = existsSync(join(benchRoot, "api-reference.md")) - ? readFileSync(join(benchRoot, "api-reference.md"), "utf8") - : ""; - - const systemPrompt = `You are a CAD engineer writing ForgeCAD scripts. ForgeCAD uses JavaScript with a globally available API (no imports needed). Scripts must return geometry via a \`return\` statement. - -${apiRef} - -IMPORTANT RULES: -- Return ONLY the JavaScript code — no markdown fences, no explanation -- The script must end with a \`return\` statement -- All API functions are global (no imports) -- Coordinate system: Z-up, millimeters, degrees`; - - const userPrompt = `${prompt} - -Write the ForgeCAD script. Return ONLY the JavaScript code, nothing else.`; - - if (verbose) console.log(` Calling ${model}...`); - - const response = await fetch("https://openrouter.ai/api/v1/chat/completions", { - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - model, - messages: [ - { role: "system", content: systemPrompt }, - { role: "user", content: userPrompt }, - ], - temperature: 0.3, - max_tokens: 4096, - }), - }); - - if (!response.ok) { - const text = await response.text(); - throw new Error(`OpenRouter API error ${response.status}: ${text}`); - } - - const data = await response.json(); - let code = data.choices?.[0]?.message?.content || ""; - - // Strip markdown fences if the model wrapped the code - // Try to extract code from ```...``` block first - const fenceMatch = code.match(/```(?:javascript|js)?\s*\n([\s\S]*?)```/); - if (fenceMatch) { - code = fenceMatch[1].trim(); - } else { - // Fallback: strip any leading/trailing fences - code = code.replace(/^```(?:javascript|js)?\s*\n?/gm, "").replace(/\n?```\s*$/gm, "").trim(); - } - - if (verbose) { - console.log(` Generated ${code.length} chars`); - console.log(` Tokens: ${data.usage?.total_tokens || "?"}`); - if (code.length === 0) { - console.log(` Raw response (first 500 chars): ${(data.choices?.[0]?.message?.content || "").slice(0, 500)}`); - } - } - - return code; -} - -// --------------------------------------------------------------------------- -// Run a harness and parse the score -// --------------------------------------------------------------------------- - -function runHarness(task) { - const harness = join(task, "harness.forge.js"); - - try { - const output = execSync(`forgecad run "${harness}" 2>&1`, { - encoding: "utf8", - timeout: 30000, - cwd: process.cwd(), - }); - - // Parse score from output: "SCORE: 8/10 (80%)" - const scoreMatch = output.match(/SCORE:\s*(\d+)\/(\d+)\s*\((\d+)%\)/); - const passed = scoreMatch ? parseInt(scoreMatch[1]) : 0; - const total = scoreMatch ? parseInt(scoreMatch[2]) : 0; - const pct = scoreMatch ? parseInt(scoreMatch[3]) : 0; - - // Extract individual test results - const tests = []; - const testLines = output.match(/║\s+[✓✗]\s+T\d+:.*/g) || []; - for (const line of testLines) { - const pass = line.includes("✓"); - const name = line.replace(/║\s+[✓✗]\s+/, "").replace(/ — .*/, "").trim(); - const failMsg = line.includes(" — ") ? line.split(" — ").slice(1).join(" — ").trim() : null; - tests.push({ name, pass, failMsg }); - } - - return { passed, total, pct, tests, output, error: null }; - } catch (e) { - // Script crashed — extract error - const output = e.stdout || e.stderr || e.message; - const errorMatch = output.match(/ERROR:\s*(.*?)(?:\n|$)/); - return { - passed: 0, - total: 0, - pct: 0, - tests: [], - output, - error: errorMatch ? errorMatch[1] : "Script crashed", - }; - } -} - -// --------------------------------------------------------------------------- -// Run a single task -// --------------------------------------------------------------------------- - -async function runTask(task, model, solution) { - const taskName = basename(task); - const solutionFile = join(task, "solution.forge.js"); - let generated = false; - - try { - // Place solution file - if (solution) { - copyFileSync(solution, solutionFile); - } else if (model) { - const code = await generateSolution(task, model); - writeFileSync(solutionFile, code, "utf8"); - generated = true; - } else { - console.error(` No solution or model specified for ${taskName}`); - return null; - } - - // Run harness - const result = runHarness(task); - result.taskName = taskName; - result.model = model || "local"; - - if (verbose || !runAll) { - // Print the scorecard from the harness output - const scoreLines = (result.output || "").split("\n").filter((l) => l.includes("[warn]")); - for (const line of scoreLines) { - console.log(line.replace(/.*\[warn\]\s*/, " ")); - } - } - - return result; - } finally { - // Save generated solution for debugging, clean up otherwise - if (generated && existsSync(solutionFile)) { - const debugPath = join(task, `solution-${(model || "unknown").replace(/\//g, "_")}.forge.js`); - copyFileSync(solutionFile, debugPath); - unlinkSync(solutionFile); - if (verbose) console.log(` Saved to ${debugPath}`); - } - } -} - -// --------------------------------------------------------------------------- -// Main -// --------------------------------------------------------------------------- - -async function main() { - const tasks = runAll ? discoverTasks() : [resolve(taskDir)]; - - if (tasks.length === 0) { - console.error("No benchmark tasks found"); - process.exit(1); - } - - console.log(`\nForgeCAD Bench — ${modelId || "local solution"}`); - console.log(`Tasks: ${tasks.map((t) => basename(t)).join(", ")}\n`); - - let allResults = []; - - if (parallel && modelId) { - // Parallel mode: generate all solutions concurrently, then run harnesses - console.log(` (parallel mode — generating all solutions concurrently)\n`); - const promises = tasks.map(async (task) => { - const taskName = basename(task); - try { - const result = await runTask(task, modelId, solutionPath); - return result; - } catch (e) { - return { taskName, passed: 0, total: 0, pct: 0, tests: [], output: "", error: e.message }; - } - }); - const settled = await Promise.all(promises); - for (const result of settled) { - if (result) { - allResults.push(result); - const name = result.taskName || "?"; - console.log(`─── ${name} ───`); - if (result.error) { - console.log(` ERROR: ${result.error}\n`); - } else { - console.log(` Score: ${result.passed}/${result.total} (${result.pct}%)\n`); - } - } - } - } else { - // Sequential mode - for (const task of tasks) { - const taskName = basename(task); - console.log(`─── ${taskName} ───`); - - const result = await runTask(task, modelId, solutionPath); - if (result) { - allResults.push(result); - if (result.error) { - console.log(` ERROR: ${result.error}\n`); - } else { - console.log(` Score: ${result.passed}/${result.total} (${result.pct}%)\n`); - } - } - } - } - - // Summary table - if (allResults.length > 1) { - console.log("═══════════════════════════════════════════"); - console.log(" BENCHMARK SUMMARY"); - console.log("───────────────────────────────────────────"); - let totalPassed = 0; - let totalTests = 0; - for (const r of allResults) { - const status = r.error ? `ERROR: ${r.error}` : `${r.passed}/${r.total} (${r.pct}%)`; - console.log(` ${r.taskName.padEnd(22)} ${status}`); - totalPassed += r.passed; - totalTests += r.total; - } - console.log("───────────────────────────────────────────"); - const totalPct = totalTests > 0 ? Math.round((100 * totalPassed) / totalTests) : 0; - console.log(` ${"TOTAL".padEnd(22)} ${totalPassed}/${totalTests} (${totalPct}%)`); - console.log("═══════════════════════════════════════════"); - } -} - -main().catch((e) => { - console.error("Fatal:", e.message); - process.exit(1); -}); diff --git a/bench/runner.mjs b/bench/runner.mjs new file mode 100644 index 00000000..5e0a97f2 --- /dev/null +++ b/bench/runner.mjs @@ -0,0 +1,599 @@ +#!/usr/bin/env node +/** + * ForgeCAD Benchmark Runner — Vision-Based Agentic Loop + * + * The AI sees multi-angle images of a reference 3D model, writes ForgeCAD code + * to reproduce it, receives spatial scores and rendered comparison images, and + * iterates. All iterations are captured for analysis. + * + * Usage: + * node bench/runner.mjs --challenge bench/challenges/cup --model claude-sonnet-4-20250514 --iterations 5 + * node bench/runner.mjs --challenge bench/challenges/cup --model openai/gpt-5.4-mini --provider openrouter + * node bench/runner.mjs --all --model claude-sonnet-4-20250514 --iterations 3 + * + * Environment: + * ANTHROPIC_API_KEY — required for Anthropic provider (default) + * OPENROUTER_API_KEY — required for OpenRouter provider + */ + +import { + readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, copyFileSync, unlinkSync, +} from 'fs'; +import { execSync } from 'child_process'; +import { join, resolve, dirname, basename } from 'path'; +import { fileURLToPath } from 'url'; +import { scoreSolution } from './score.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PROJECT_ROOT = resolve(__dirname, '..'); +const FORGECAD_CLI = join(PROJECT_ROOT, 'dist-cli', 'forgecad.js'); +const CHALLENGES_DIR = join(__dirname, 'challenges'); +const RESULTS_DIR = join(__dirname, 'results'); + +// --------------------------------------------------------------------------- +// API reference docs for the LLM +// --------------------------------------------------------------------------- + +function loadApiReference() { + for (const p of [ + join(__dirname, 'api-reference.md'), + join(__dirname, '..', 'examples', 'bench-v2', 'api-reference.md'), + ]) { + if (existsSync(p)) return readFileSync(p, 'utf8'); + } + return ''; +} +const API_REFERENCE = loadApiReference(); + +// --------------------------------------------------------------------------- +// Image helpers +// --------------------------------------------------------------------------- + +function loadImageBase64(path) { + if (!existsSync(path)) return null; + return readFileSync(path).toString('base64'); +} + +function imageContent(base64, label) { + return [ + { type: 'text', text: label }, + { + type: 'image_url', + image_url: { url: `data:image/png;base64,${base64}` }, + }, + ]; +} + +/** Load all reference view images from a challenge's prepared/views/ directory */ +function loadReferenceImages(challengeDir) { + const viewsDir = join(challengeDir, 'prepared', 'views'); + if (!existsSync(viewsDir)) return []; + + const images = []; + for (const angle of ['front', 'right', 'top', 'iso']) { + const path = join(viewsDir, `reference_${angle}.png`); + const b64 = loadImageBase64(path); + if (b64) images.push({ angle, base64: b64 }); + } + return images; +} + +/** Render candidate images and return base64 data */ +function renderCandidate(challengeDir, solutionCode, outputDir, imageSize = 512) { + const solutionFile = join(challengeDir, '_bench_candidate.forge.js'); + writeFileSync(solutionFile, solutionCode, 'utf8'); + + const outputBase = join(outputDir, 'candidate.png'); + try { + execSync( + `node "${FORGECAD_CLI}" render "${solutionFile}" "${outputBase}" --angles front,right,top,iso --size ${imageSize} 2>&1`, + { encoding: 'utf8', timeout: 120_000, cwd: PROJECT_ROOT }, + ); + } catch (e) { + console.warn(` Warning: Candidate rendering failed: ${e.message.split('\n')[0]}`); + return []; + } finally { + try { unlinkSync(solutionFile); } catch {} + } + + const images = []; + for (const angle of ['front', 'right', 'top', 'iso']) { + const path = join(outputDir, `candidate_${angle}.png`); + const b64 = loadImageBase64(path); + if (b64) images.push({ angle, base64: b64 }); + } + return images; +} + +// --------------------------------------------------------------------------- +// LLM API clients +// --------------------------------------------------------------------------- + +/** + * Call the Anthropic Messages API. + * @param {string} model e.g. "claude-sonnet-4-20250514" + * @param {Array} messages Anthropic message format + * @param {string} system System prompt + * @returns {Promise<{text: string, usage: object}>} + */ +async function callAnthropic(model, messages, system) { + const apiKey = process.env.ANTHROPIC_API_KEY; + if (!apiKey) throw new Error('ANTHROPIC_API_KEY not set'); + + const body = { + model, + max_tokens: 8192, + system, + messages: messages.map(m => ({ + role: m.role, + content: Array.isArray(m.content) ? m.content.map(c => { + if (c.type === 'image_url') { + // Convert OpenAI-style image to Anthropic format + const match = c.image_url.url.match(/^data:(.+?);base64,(.+)$/); + if (match) { + return { + type: 'image', + source: { type: 'base64', media_type: match[1], data: match[2] }, + }; + } + } + return c; + }) : m.content, + })), + }; + + const resp = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + 'content-type': 'application/json', + }, + body: JSON.stringify(body), + }); + + if (!resp.ok) { + const text = await resp.text(); + throw new Error(`Anthropic API ${resp.status}: ${text}`); + } + + const data = await resp.json(); + const text = data.content?.map(b => b.text || '').join('') || ''; + return { text, usage: data.usage }; +} + +/** + * Call OpenRouter API (supports any model). + */ +async function callOpenRouter(model, messages, system) { + const apiKey = process.env.OPENROUTER_API_KEY; + if (!apiKey) throw new Error('OPENROUTER_API_KEY not set'); + + const body = { + model, + messages: [ + { role: 'system', content: system }, + ...messages, + ], + temperature: 0.3, + max_tokens: 8192, + }; + + const resp = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }); + + if (!resp.ok) { + const text = await resp.text(); + throw new Error(`OpenRouter API ${resp.status}: ${text}`); + } + + const data = await resp.json(); + const text = data.choices?.[0]?.message?.content || ''; + return { text, usage: data.usage }; +} + +// --------------------------------------------------------------------------- +// Code extraction +// --------------------------------------------------------------------------- + +function extractCode(llmResponse) { + // Try fenced code block first + const fenceMatch = llmResponse.match(/```(?:javascript|js)?\s*\n([\s\S]*?)```/); + if (fenceMatch) return fenceMatch[1].trim(); + + // Try to find return statement — the code is likely the whole response + const stripped = llmResponse + .replace(/^```(?:javascript|js)?\s*\n?/gm, '') + .replace(/\n?```\s*$/gm, '') + .trim(); + + if (stripped.includes('return ')) return stripped; + + return llmResponse.trim(); +} + +// --------------------------------------------------------------------------- +// Prompt construction +// --------------------------------------------------------------------------- + +function buildSystemPrompt(apiReference) { + return `You are a CAD engineer writing ForgeCAD scripts. You are given reference images of a 3D model from multiple angles (front, right, top, isometric) and must write ForgeCAD JavaScript code that reproduces the geometry as accurately as possible. + +${apiReference} + +IMPORTANT RULES: +- Return ONLY the JavaScript code inside a single \`\`\`javascript code fence +- The script must end with a \`return\` statement returning the final Shape +- All API functions are global (no imports needed) +- Coordinate system: Z-up, millimeters, degrees +- Study the reference images carefully — match dimensions, proportions, and orientation +- Pay attention to holes, cutouts, fillets, and other features visible in the images`; +} + +function buildInitialUserContent(referenceImages, config) { + const content = []; + + content.push({ + type: 'text', + text: `# Task: Reproduce this 3D model in ForgeCAD + +Study the reference images below carefully. They show a "${config.name}" model from four angles: front, right, top, and isometric. + +Write ForgeCAD code that reproduces this geometry as accurately as possible. Pay close attention to: +- Overall shape and proportions +- Holes, cutouts, and internal features +- Orientation (Z is up) +- Approximate dimensions (estimate from proportions in the images)`, + }); + + for (const img of referenceImages) { + content.push({ type: 'text', text: `Reference — ${img.angle} view:` }); + content.push({ + type: 'image_url', + image_url: { url: `data:image/png;base64,${img.base64}` }, + }); + } + + // Include metrics hint if available + if (config.metrics) { + const m = config.metrics; + content.push({ + type: 'text', + text: `\nDimension hints:\n- Bounding box: ${m.boundingBox.size[0].toFixed(1)} × ${m.boundingBox.size[1].toFixed(1)} × ${m.boundingBox.size[2].toFixed(1)} mm\n- Volume: ${m.volume.toFixed(1)} mm³\n- Surface area: ${m.surfaceArea.toFixed(1)} mm²`, + }); + } + + return content; +} + +function buildIterationUserContent(score, candidateImages, referenceImages, iterationNum, maxIterations) { + const content = []; + + content.push({ + type: 'text', + text: `# Iteration ${iterationNum} Feedback + +Your previous attempt scored **${score.overall.toFixed(1)}%** overall. + +Score breakdown: +- 3D IoU: ${((score.breakdown?.iou?.score || 0) * 100).toFixed(1)}% (volumetric overlap between your model and reference) +- Volume: ${((score.breakdown?.volume?.score || 0) * 100).toFixed(1)}% (ref=${score.breakdown?.volume?.ref || '?'} mm³, yours=${score.breakdown?.volume?.cand || '?'} mm³) +- Bounding box: ${((score.breakdown?.bbox?.score || 0) * 100).toFixed(1)}% (ref=${JSON.stringify(score.breakdown?.bbox?.ref)}, yours=${JSON.stringify(score.breakdown?.bbox?.cand)}) +- Surface area: ${((score.breakdown?.surfaceArea?.score || 0) * 100).toFixed(1)}% (ref=${score.breakdown?.surfaceArea?.ref || '?'} mm², yours=${score.breakdown?.surfaceArea?.cand || '?'} mm²) +${score.error ? `\nError: ${score.error}\n` : ''} +You have ${maxIterations - iterationNum} iteration(s) remaining. Compare your rendered output below against the reference images and improve your code.`, + }); + + // Show candidate renders side-by-side with reference + for (const angle of ['front', 'right', 'top', 'iso']) { + const candImg = candidateImages.find(i => i.angle === angle); + const refImg = referenceImages.find(i => i.angle === angle); + + if (refImg) { + content.push({ type: 'text', text: `Reference — ${angle}:` }); + content.push({ + type: 'image_url', + image_url: { url: `data:image/png;base64,${refImg.base64}` }, + }); + } + if (candImg) { + content.push({ type: 'text', text: `Your output — ${angle}:` }); + content.push({ + type: 'image_url', + image_url: { url: `data:image/png;base64,${candImg.base64}` }, + }); + } + } + + content.push({ + type: 'text', + text: 'Write the improved ForgeCAD code. Return ONLY the code in a ```javascript fence.', + }); + + return content; +} + +// --------------------------------------------------------------------------- +// Main benchmark loop +// --------------------------------------------------------------------------- + +/** + * Run the benchmark for a single challenge. + * + * @param {string} challengeDir + * @param {object} options + * @param {string} options.model LLM model ID + * @param {string} [options.provider] 'anthropic' | 'openrouter' (default: 'anthropic') + * @param {number} [options.iterations] Max iterations (default 5) + * @param {boolean} [options.verbose] + * @param {number} [options.imageSize] Render size (default 512) + * @returns {Promise} Run results + */ +export async function runBenchmark(challengeDir, options) { + const { + model, + provider = 'anthropic', + iterations: maxIterations = 5, + verbose = false, + imageSize = 512, + } = options; + + const absChallenge = resolve(challengeDir); + const configPath = join(absChallenge, 'config.json'); + if (!existsSync(configPath)) { + throw new Error(`No config.json in ${absChallenge}. Run bench/prepare.mjs first.`); + } + + const config = JSON.parse(readFileSync(configPath, 'utf8')); + const challengeName = config.name || basename(absChallenge); + + // Load reference images + const referenceImages = loadReferenceImages(absChallenge); + if (referenceImages.length === 0) { + throw new Error(`No reference images found in ${absChallenge}/prepared/views/. Run bench/prepare.mjs first.`); + } + + // Load metrics if available + const metricsPath = join(absChallenge, 'prepared', 'metrics.json'); + if (existsSync(metricsPath)) { + config.metrics = JSON.parse(readFileSync(metricsPath, 'utf8')); + } + + // Create results directory + const runId = `${challengeName}-${model.replace(/[/:.]/g, '_')}-${Date.now()}`; + const runDir = join(RESULTS_DIR, runId); + mkdirSync(runDir, { recursive: true }); + + console.log(`\n${'═'.repeat(60)}`); + console.log(` ForgeCAD Benchmark — ${challengeName}`); + console.log(` Model: ${model} (${provider})`); + console.log(` Max iterations: ${maxIterations}`); + console.log(`${'═'.repeat(60)}\n`); + + const llmCall = provider === 'openrouter' ? callOpenRouter : callAnthropic; + const systemPrompt = buildSystemPrompt(API_REFERENCE); + const apiReference = API_REFERENCE; + + const messages = []; // Conversation history + const iterationResults = []; // All iteration data + let bestScore = null; + let bestCode = null; + + for (let i = 0; i < maxIterations; i++) { + const iterDir = join(runDir, 'iterations', String(i)); + mkdirSync(join(iterDir, 'views'), { recursive: true }); + + console.log(`── Iteration ${i} ──`); + + // Build prompt + let userContent; + if (i === 0) { + userContent = buildInitialUserContent(referenceImages, config); + } else { + const prevResult = iterationResults[i - 1]; + userContent = buildIterationUserContent( + prevResult.score, prevResult.candidateImages, referenceImages, i, maxIterations, + ); + } + + messages.push({ role: 'user', content: userContent }); + + // Call LLM + console.log(` Calling ${model}...`); + const startTime = Date.now(); + let llmResult; + try { + llmResult = await llmCall(model, messages, systemPrompt); + } catch (e) { + console.error(` LLM error: ${e.message}`); + iterationResults.push({ iteration: i, error: `LLM error: ${e.message}`, score: { overall: 0 } }); + break; + } + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + + // Extract code + const code = extractCode(llmResult.text); + writeFileSync(join(iterDir, 'code.forge.js'), code, 'utf8'); + writeFileSync(join(iterDir, 'llm-response.txt'), llmResult.text, 'utf8'); + writeFileSync(join(iterDir, 'llm-usage.json'), JSON.stringify(llmResult.usage, null, 2), 'utf8'); + + // Add assistant response to conversation + messages.push({ role: 'assistant', content: llmResult.text }); + + // Score the solution + console.log(` Scoring... (LLM took ${elapsed}s)`); + const score = scoreSolution(absChallenge, code, { verbose }); + writeFileSync(join(iterDir, 'score.json'), JSON.stringify(score, null, 2), 'utf8'); + + // Render candidate images + const candidateImages = renderCandidate(absChallenge, code, join(iterDir, 'views'), imageSize); + + const iterData = { + iteration: i, + score, + codeLength: code.length, + llmTime: parseFloat(elapsed), + usage: llmResult.usage, + candidateImages, + }; + iterationResults.push(iterData); + + // Print score + if (score.error) { + console.log(` Error: ${score.error}`); + } + console.log(` Score: ${score.overall.toFixed(1)}%`); + if (score.breakdown) { + const b = score.breakdown; + console.log(` IoU=${((b.iou?.score || 0) * 100).toFixed(0)}% Vol=${((b.volume?.score || 0) * 100).toFixed(0)}% BBox=${((b.bbox?.score || 0) * 100).toFixed(0)}% SA=${((b.surfaceArea?.score || 0) * 100).toFixed(0)}%`); + } + + // Track best + if (!bestScore || score.overall > bestScore.overall) { + bestScore = score; + bestCode = code; + } + + // Early termination if perfect score + if (score.overall >= 95) { + console.log(` Score ≥ 95% — stopping early.`); + break; + } + } + + // Save best solution + if (bestCode) { + writeFileSync(join(runDir, 'best-solution.forge.js'), bestCode, 'utf8'); + } + + // Save run summary + const summary = { + challenge: challengeName, + model, + provider, + maxIterations, + completedIterations: iterationResults.length, + bestScore: bestScore?.overall || 0, + finalScore: iterationResults[iterationResults.length - 1]?.score?.overall || 0, + scoreProgression: iterationResults.map(r => r.score?.overall || 0), + totalLlmTime: iterationResults.reduce((s, r) => s + (r.llmTime || 0), 0), + timestamp: new Date().toISOString(), + iterationSummaries: iterationResults.map(r => ({ + iteration: r.iteration, + score: r.score?.overall || 0, + breakdown: r.score?.breakdown, + error: r.score?.error || r.error, + codeLength: r.codeLength, + llmTime: r.llmTime, + })), + }; + writeFileSync(join(runDir, 'summary.json'), JSON.stringify(summary, null, 2), 'utf8'); + + // Print final summary + console.log(`\n${'═'.repeat(60)}`); + console.log(` FINAL RESULTS — ${challengeName}`); + console.log(`${'─'.repeat(60)}`); + console.log(` Model: ${model}`); + console.log(` Iterations: ${iterationResults.length}/${maxIterations}`); + console.log(` Best score: ${(bestScore?.overall || 0).toFixed(1)}%`); + console.log(` Progression: ${summary.scoreProgression.map(s => s.toFixed(1) + '%').join(' → ')}`); + console.log(` Results: ${runDir}`); + console.log(`${'═'.repeat(60)}\n`); + + return summary; +} + +// --------------------------------------------------------------------------- +// Discover challenges +// --------------------------------------------------------------------------- + +function discoverChallenges() { + if (!existsSync(CHALLENGES_DIR)) return []; + return readdirSync(CHALLENGES_DIR, { withFileTypes: true }) + .filter(d => d.isDirectory()) + .map(d => join(CHALLENGES_DIR, d.name)) + .filter(d => existsSync(join(d, 'config.json')) && existsSync(join(d, 'reference.forge.js'))); +} + +// --------------------------------------------------------------------------- +// CLI entry point +// --------------------------------------------------------------------------- + +const isMainModule = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMainModule) { + const args = process.argv.slice(2); + + const getArg = (name) => { + const idx = args.indexOf(`--${name}`); + return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : null; + }; + const hasFlag = (name) => args.includes(`--${name}`); + + const challengeDir = getArg('challenge'); + const model = getArg('model'); + const provider = getArg('provider') || 'anthropic'; + const iterations = parseInt(getArg('iterations') || '5'); + const runAll = hasFlag('all'); + const verbose = hasFlag('verbose'); + const imageSize = parseInt(getArg('size') || '512'); + + if (!model) { + console.error('Usage: node bench/runner.mjs --challenge --model [options]'); + console.error(' node bench/runner.mjs --all --model [options]'); + console.error(''); + console.error('Options:'); + console.error(' --challenge Challenge directory'); + console.error(' --model LLM model ID (e.g. claude-sonnet-4-20250514, openai/gpt-5.4-mini)'); + console.error(' --provider API provider: anthropic (default) or openrouter'); + console.error(' --iterations Max iterations (default 5)'); + console.error(' --size Image render size (default 512)'); + console.error(' --all Run all prepared challenges'); + console.error(' --verbose Show harness output'); + console.error(''); + console.error('Environment:'); + console.error(' ANTHROPIC_API_KEY Required for --provider anthropic'); + console.error(' OPENROUTER_API_KEY Required for --provider openrouter'); + process.exit(1); + } + + const challenges = runAll ? discoverChallenges() : (challengeDir ? [resolve(challengeDir)] : []); + if (challenges.length === 0) { + console.error('No challenges found. Specify --challenge or prepare challenges first.'); + process.exit(1); + } + + const allSummaries = []; + + for (const challenge of challenges) { + try { + const summary = await runBenchmark(challenge, { model, provider, iterations, verbose, imageSize }); + allSummaries.push(summary); + } catch (e) { + console.error(`Error on ${basename(challenge)}: ${e.message}`); + allSummaries.push({ challenge: basename(challenge), error: e.message, bestScore: 0 }); + } + } + + // Multi-challenge summary + if (allSummaries.length > 1) { + console.log(`\n${'═'.repeat(60)}`); + console.log(' BENCHMARK SUMMARY'); + console.log(`${'─'.repeat(60)}`); + for (const s of allSummaries) { + const name = (s.challenge || '?').padEnd(25); + const score = s.error ? `ERROR: ${s.error}` : `${s.bestScore.toFixed(1)}%`; + console.log(` ${name} ${score}`); + } + const avg = allSummaries.filter(s => !s.error).reduce((a, s) => a + s.bestScore, 0) + / Math.max(allSummaries.filter(s => !s.error).length, 1); + console.log(`${'─'.repeat(60)}`); + console.log(` Average best score: ${avg.toFixed(1)}%`); + console.log(`${'═'.repeat(60)}\n`); + } +} diff --git a/bench/score.mjs b/bench/score.mjs new file mode 100644 index 00000000..4ebc7c6c --- /dev/null +++ b/bench/score.mjs @@ -0,0 +1,237 @@ +#!/usr/bin/env node +/** + * ForgeCAD Benchmark — Scoring Module + * + * Scores a candidate .forge.js solution against a reference model using + * spatial validators: 3D IoU, volume match, bounding-box match, surface-area match. + * + * Can be used standalone or imported by the runner. + * + * Usage: + * node bench/score.mjs + * + * The challenge directory must contain a reference.forge.js file. + */ + +import { readFileSync, writeFileSync, unlinkSync, existsSync, mkdirSync } from 'fs'; +import { execSync } from 'child_process'; +import { join, resolve, dirname, basename, extname } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const PROJECT_ROOT = resolve(__dirname, '..'); +const FORGECAD_CLI = join(PROJECT_ROOT, 'dist-cli', 'forgecad.js'); + +// --------------------------------------------------------------------------- +// Scoring harness — a .forge.js script that imports both reference and +// candidate, computes spatial similarity metrics, and outputs structured JSON. +// --------------------------------------------------------------------------- + +const SCORING_HARNESS = ` +// Auto-generated scoring harness — do not edit +// Compares candidate (solution.forge.js) against reference (reference.forge.js) + +function round(v) { return Math.round(v * 100) / 100; } + +// ---- Load candidate (with error handling) ---- +let candidate = null; +let loadError = null; +try { + candidate = require("./solution.forge.js"); +} catch (e) { + loadError = e.message || String(e); +} + +if (loadError || !candidate || candidate.isEmpty()) { + const result = { + error: loadError || "Empty or null shape returned", + overall: 0, + breakdown: { + iou: { score: 0, weight: 0.50 }, + volume: { score: 0, weight: 0.20 }, + bbox: { score: 0, weight: 0.15 }, + surfaceArea: { score: 0, weight: 0.15 }, + }, + candidateMetrics: null, + referenceMetrics: null, + }; + console.warn("BENCH_SCORE:" + JSON.stringify(result)); + return box(1, 1, 1); // must return something +} + +// ---- Load reference ---- +const reference = require("./reference.forge.js"); + +// ---- Gather metrics ---- +const refVol = reference.volume(); +const candVol = candidate.volume(); +const refSA = reference.surfaceArea(); +const candSA = candidate.surfaceArea(); +const refBB = reference.boundingBox(); +const candBB = candidate.boundingBox(); + +const refSize = [refBB.max[0]-refBB.min[0], refBB.max[1]-refBB.min[1], refBB.max[2]-refBB.min[2]]; +const candSize = [candBB.max[0]-candBB.min[0], candBB.max[1]-candBB.min[1], candBB.max[2]-candBB.min[2]]; + +// Sort dimensions descending for orientation-independent comparison +const refSorted = [...refSize].sort((a, b) => b - a); +const candSorted = [...candSize].sort((a, b) => b - a); + +// ---- Validator 1: Volume similarity ---- +const volScore = Math.min(refVol, candVol) / Math.max(refVol, candVol || 0.001); + +// ---- Validator 2: Bounding-box similarity (orientation-independent) ---- +let bboxErr = 0, bboxSum = 0; +for (let i = 0; i < 3; i++) { + bboxErr += Math.abs(refSorted[i] - candSorted[i]); + bboxSum += refSorted[i]; +} +const bboxScore = Math.max(0, 1 - bboxErr / (bboxSum || 0.001)); + +// ---- Validator 3: Surface-area similarity ---- +const saScore = Math.min(refSA, candSA) / Math.max(refSA, candSA || 0.001); + +// ---- Validator 4: 3D IoU (Intersection over Union) ---- +// Center both shapes at origin to remove trivial translation offset, +// but preserve orientation (the AI should match orientation from images). +let iouScore = 0; +let iouError = null; +try { + const refC = [(refBB.min[0]+refBB.max[0])/2, (refBB.min[1]+refBB.max[1])/2, (refBB.min[2]+refBB.max[2])/2]; + const candC = [(candBB.min[0]+candBB.max[0])/2, (candBB.min[1]+candBB.max[1])/2, (candBB.min[2]+candBB.max[2])/2]; + + const refCentered = reference.translate(-refC[0], -refC[1], -refC[2]); + const candCentered = candidate.translate(-candC[0], -candC[1], -candC[2]); + + const inter = intersection(refCentered, candCentered); + const uni = union(refCentered, candCentered); + + const interVol = inter.isEmpty() ? 0 : inter.volume(); + const uniVol = uni.isEmpty() ? 0.001 : uni.volume(); + iouScore = interVol / uniVol; +} catch (e) { + iouError = e.message || String(e); +} + +// ---- Weighted overall score ---- +const W = { iou: 0.50, volume: 0.20, bbox: 0.15, surfaceArea: 0.15 }; +const overall = W.iou * iouScore + W.volume * volScore + W.bbox * bboxScore + W.surfaceArea * saScore; + +const result = { + overall: Math.round(overall * 1000) / 10, // percentage 0-100 + breakdown: { + iou: { score: round(iouScore), weight: W.iou, error: iouError }, + volume: { score: round(volScore), weight: W.volume, ref: round(refVol), cand: round(candVol) }, + bbox: { score: round(bboxScore), weight: W.bbox, ref: refSorted.map(round), cand: candSorted.map(round) }, + surfaceArea: { score: round(saScore), weight: W.surfaceArea, ref: round(refSA), cand: round(candSA) }, + }, + referenceMetrics: { + volume: round(refVol), + surfaceArea: round(refSA), + bbox: { size: refSize.map(round), min: refBB.min, max: refBB.max }, + }, + candidateMetrics: { + volume: round(candVol), + surfaceArea: round(candSA), + bbox: { size: candSize.map(round), min: candBB.min, max: candBB.max }, + }, +}; + +// ---- Pretty-print scorecard ---- +console.warn(""); +console.warn("\\u2554\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2557"); +console.warn("\\u2551 SPATIAL SCORING \\u2551"); +console.warn("\\u255f\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2562"); +console.warn("\\u2551 3D IoU: " + (iouScore*100).toFixed(1).padStart(5) + "% (weight " + W.iou + ")" + (iouError ? " ERR" : "") + " \\u2551"); +console.warn("\\u2551 Volume match: " + (volScore*100).toFixed(1).padStart(5) + "% (weight " + W.volume + ")" + " \\u2551"); +console.warn("\\u2551 BBox match: " + (bboxScore*100).toFixed(1).padStart(5) + "% (weight " + W.bbox + ")" + " \\u2551"); +console.warn("\\u2551 Surface area: " + (saScore*100).toFixed(1).padStart(5) + "% (weight " + W.surfaceArea + ")" + " \\u2551"); +console.warn("\\u255f\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2500\\u2562"); +console.warn("\\u2551 OVERALL: " + result.overall.toFixed(1) + "%" + " \\u2551"); +console.warn("\\u255a\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u2550\\u255d"); +console.warn("BENCH_SCORE:" + JSON.stringify(result)); + +return candidate; +`; + +// --------------------------------------------------------------------------- +// Score a solution against a reference +// --------------------------------------------------------------------------- + +/** + * @param {string} challengeDir Path to the challenge directory (must contain reference.forge.js) + * @param {string} solutionCode The candidate ForgeCAD code (string, not file path) + * @param {object} [options] + * @param {boolean} [options.verbose] Print harness output + * @param {number} [options.timeout] Execution timeout in ms (default 60 000) + * @returns {{ overall: number, breakdown: object, error?: string, output?: string }} + */ +export function scoreSolution(challengeDir, solutionCode, options = {}) { + const absChallenge = resolve(challengeDir); + const refFile = join(absChallenge, 'reference.forge.js'); + if (!existsSync(refFile)) { + return { overall: 0, error: `No reference.forge.js in ${absChallenge}`, breakdown: {} }; + } + + const solutionFile = join(absChallenge, 'solution.forge.js'); + const harnessFile = join(absChallenge, '_bench_harness.forge.js'); + + writeFileSync(solutionFile, solutionCode, 'utf8'); + writeFileSync(harnessFile, SCORING_HARNESS, 'utf8'); + + try { + const output = execSync( + `node "${FORGECAD_CLI}" run "${harnessFile}" 2>&1`, + { encoding: 'utf8', timeout: options.timeout || 60_000, cwd: PROJECT_ROOT }, + ); + + if (options.verbose) process.stderr.write(output); + + const match = output.match(/BENCH_SCORE:(.+)/); + if (!match) return { overall: 0, error: 'No BENCH_SCORE in harness output', output }; + + const result = JSON.parse(match[1]); + result.output = output; + return result; + } catch (e) { + const output = (e.stdout || '') + (e.stderr || '') + (e.message || ''); + return { overall: 0, error: `Harness crashed: ${e.message}`, output, breakdown: {} }; + } finally { + try { unlinkSync(solutionFile); } catch {} + try { unlinkSync(harnessFile); } catch {} + } +} + +/** + * Score from a file path instead of inline code. + */ +export function scoreSolutionFile(challengeDir, solutionPath, options = {}) { + const code = readFileSync(resolve(solutionPath), 'utf8'); + return scoreSolution(challengeDir, code, options); +} + +// --------------------------------------------------------------------------- +// CLI entry point +// --------------------------------------------------------------------------- + +const isMainModule = process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url)); +if (isMainModule) { + const args = process.argv.slice(2); + const verbose = args.includes('--verbose'); + const positional = args.filter(a => !a.startsWith('--')); + + if (positional.length < 2) { + console.error('Usage: node bench/score.mjs [--verbose]'); + process.exit(1); + } + + const [challengeDir, solutionPath] = positional; + const result = scoreSolutionFile(challengeDir, solutionPath, { verbose }); + + // Pretty summary + if (result.error) { + console.error(`\nError: ${result.error}\n`); + } + console.log(JSON.stringify(result, null, 2)); + process.exit(result.overall > 0 ? 0 : 1); +}