From 8db5cfebd4b23d00b981a482237a61bdf65949a3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 16:47:13 +0000 Subject: [PATCH 1/2] feat(fs): record lstatSync/statSync captures as payload-free stat records lstatSync/statSync (and their async forms under --fs=async) were serve-only: a path that was ONLY stat'd -- never read, readdir'd, or imported -- was not recorded, so on a bundle=load run its stat fell through to the real fs and threw ENOENT once the path was gone from disk. Capture now records the observed KIND as a payload-free stat record: formats[file] = 'stat:file' | 'stat:directory' -- no bytes, no hash, no module-files entry. That is exactly what the load-side shim serves (the type getters isFile()/isDirectory(); every other Stats member stays real-if-on-disk / benign-synthetic-once-gone), so only type getters are supported, and the existence probes (existsSync/accessSync/realpathSync) answer from the record too. The caller always receives the real Stats (and real errors) at capture; symlinks/sockets/FIFOs aren't modelled and record nothing; escaping-symlink and sidecar-attested paths are skipped like the byte readers; options forms (bigint, throwIfNoEntry) pass through uncaptured. Stat records are deliberately WEAK: real content recorded for the same path supersedes them -- in-place (addFile/addFsDir drop the stale tag, so check-then-read never conflicts), across lock=add/bundle=add re-runs (the format-flip guard and #mergedFormats upgrade a stat baseline), and at bundle serialization (a record a write-mode sidecar upgraded is dropped so the bundle keeps cross-checking against the unified lockfile). addFile refuses an explicit 'stat:*' format, and a forked child's stat records ride the shard channel: shardSnapshot forwards the formats entry and mergeShard replays it by re-deriving the kind from disk. Since getFsStat now answers 'file' for byte-less records, callers that mean "the bundle carries this file's bytes" -- the parentless CJS re-resolution shim and attestedBySidecar -- switch to the new hasFsFileContent. Implied ancestor directories are derived from stat records too, stasis build allowlists the new formats, and the loader fails closed with a dedicated message on importing a stat-only path. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017z3Eb4Udqc5BkdQtt2YA72 --- doc/file-formats.md | 54 ++-- stasis-core/src/fs.js | 101 ++++++-- stasis-core/src/hooks.js | 15 +- stasis-core/src/state.js | 178 ++++++++++++-- stasis-core/src/util.js | 16 +- stasis/src/cmd/build.js | 7 +- tests/cli-fs.test.js | 232 ++++++++++++++++++ .../cli-run-fork-shard/src/statonly.dat | 1 + .../fixtures/cli-run-fork-shard/src/worker.js | 11 + 9 files changed, 561 insertions(+), 54 deletions(-) create mode 100644 tests/fixtures/cli-run-fork-shard/src/statonly.dat diff --git a/doc/file-formats.md b/doc/file-formats.md index ef57539..000c069 100644 --- a/doc/file-formats.md +++ b/doc/file-formats.md @@ -145,7 +145,11 @@ into the lockfile/bundle `config` block (`debug`, `childProcess`, `fs` and what distinguishes a resource from code per file, and tells a reader how to decode the bundle payload. Filesystem captures (`stasis run --fs=sync`) add `directory` for a `fs.readdirSync` listing (the content is a sorted JSON array - of names); a captured `fs.readFileSync` reuses the code or resource tags above. + of names) and the **payload-free** stat records `stat:file` / `stat:directory` + for an `fs.lstatSync`/`fs.statSync` of a path that was never read, listed, or + imported — those attest only that the path existed with that kind, so they + appear *only* here in `formats`, never in a module's hashed `files`; a captured + `fs.readFileSync` reuses the code or resource tags above. The integrity of a `directory` listing is the sha512 of its JSON text, just like any other content. The loader picks module-vs-commonjs and (for the `*-typescript` formats) whether type-shaped syntax is stripped purely from @@ -214,7 +218,10 @@ reporting failures) still persists what it cleanly captured. Resource files use `resource` (raw UTF-8 payload) or `resource:base64` (base64 payload) — the per-file tag that both marks a file as a resource and says how to decode its bundle bytes. A `directory` tag (a `stasis run - --fs` capture) marks a raw-UTF-8 JSON listing. May be missing per file for code + --fs` capture) marks a raw-UTF-8 JSON listing; a `stat:file`/`stat:directory` + tag (an `--fs` `lstatSync`/`statSync` capture) is payload-free — the tagged + path has no `files` entry at all, the format alone attests its existence and + kind. May be missing per file for code whose format Node infers. TypeScript sources are stored verbatim (types intact); Node strips the types at load time based on the format. - `imports`: conditions → parent file → specifier → resolved @@ -343,12 +350,14 @@ a frozen run verifies a copied asset byte-for-byte just as it does code. ## Filesystem captures (`stasis run --fs=sync` / `--fs=async`) The loader hooks capture the module graph; `--fs=sync` additionally monkey-patches the -**sync** readers `fs.readFileSync` and `fs.readdirSync`, plus the metadata/existence probes -`fs.lstatSync`/`fs.statSync`/`fs.existsSync`/`fs.accessSync`/`fs.realpathSync` (and only +**sync** readers `fs.readFileSync` and `fs.readdirSync`, the kind-recording stat calls +`fs.lstatSync`/`fs.statSync`, plus the existence probes +`fs.existsSync`/`fs.accessSync`/`fs.realpathSync` (and only those — no streams, `fs.opendir`, `fs.readlink` (a resolver probe that already treats a read error as "not a symlink", so an absent recorded file resolves correctly unserved), the deprecated callback `fs.exists`, …) so a program's -explicit file reads are recorded into the bundle (`--bundle=add|replace`) and served back +explicit file reads — and the kind (file vs directory) of each path it stats — are +recorded into the bundle (`--bundle=add|replace`) and served back from it (`--bundle=load`). The same `--fs=…` flag is needed on the load run for the patch to serve; an un-captured read falls through to the real disk read. The mode can equivalently be set as `"fs": "sync" | "async"` in `stasis.config.json` (or `EXODUS_STASIS_FS`), which @@ -381,9 +390,21 @@ Captures live in the usual `sources`/`modules` buckets, tagged in `formats`: is not captured and is not served from the bundle on load. A listing captured *at* a package bucket root sits at the bucket's own key: rel `''` inside its `files`, format keyed at the bucket dir — `.` for the project root itself. -- `fs.lstatSync(path)` and `fs.statSync(path)` are not themselves recorded; on a load - run, for a path the bundle already carries — a recorded file, a `directory`, or an - ancestor directory implied by a recorded file (a bundled `node_modules/dep/index.js` +- A single-argument `fs.lstatSync(path)`/`fs.statSync(path)` records the path's **kind** + — and only its kind. At capture the real call runs first (the program always gets the + genuine `Stats`, and genuine errors); the observed type is then stored as a + **payload-free** stat record, `stat:file` or `stat:directory` in `formats` — no bytes, + no hash, no `files` entry — so a path that is *only* stat'd (never read, listed, or + imported) still answers its type getters on load. Only regular files and directories + are modelled: a symlink (under `lstat`), socket, or FIFO records nothing, and a + `statSync` through an in-root symlink records the *target's* kind, keyed at the + requested path exactly like a byte read of it would be. A path already carried as + content (read/listed/imported), or whose real location + escapes the root, records no stat entry; reading a stat-only path later (same run or a + `lock=add`/`bundle=add` re-run) upgrades the weak stat record to the real content + record instead of conflicting. On a load + run, for a path the bundle carries — a recorded file, a `directory`, a stat record, or + an ancestor directory implied by a recorded path (a bundled `node_modules/dep/index.js` proves `node_modules` and `node_modules/dep` are directories) — `.isFile()`/`.isDirectory()` answer from the bundle so existence checks succeed even when the path is absent from disk. Other `Stats` fields/methods are the real stat's @@ -392,13 +413,16 @@ Captures live in the usual `sources`/`modules` buckets, tagged in `formats`: `isFile()`/`isDirectory()` off the result (graceful-fs's `statFixSync` reads `uid`/`gid`, for example) keep working under `--bundle=load` instead of re-throwing `ENOENT`. Any path the bundle does not carry passes straight through to the real - call. + call. Stat calls with options (`bigint`, `throwIfNoEntry`) pass through untouched, + like `readdirSync` with options. - `fs.existsSync(path)`, `fs.accessSync(path)` and `fs.realpathSync(path)` (plus the async `fs.access`/`fs.realpath` and `fs.promises.*` forms) are **existence/canonical-path probes**, served (load run, for a path the bundle carries) and otherwise passed through. - Like `lstat`/`stat` they are **serve-only — never captured**: a probe records nothing, so a - file that is *only* existence-probed (never read or imported) is not in the bundle and its - probe falls through to disk on load. They matter because build tools check a file *before* + Unlike `lstat`/`stat` they are **serve-only — never captured**: a probe records nothing, so a + file that is *only* existence-probed (never read, stat'd, or imported) is not in the bundle + and its probe falls through to disk on load — though a path the program *does* `lstat`/`stat` + gains a stat record, from which these probes then answer too (they consult the same + kind lookup). They matter because build tools check a file *before* reading it on a channel that never touches its bytes: `@babel/core`, for one, guards config loading with `fs.existsSync(babel.config.js)` and realpath-canonicalizes the path **before** it `require()`s the file — so without serving the probe, a bundled-but-absent `babel.config.js` @@ -430,9 +454,11 @@ way. To capture a `.map` instead (e.g. you genuinely ship one as data), add `map and once a `.map` is in the bundle it is served on load by membership, even if that run omits `--resources=map`. -The two recorded kinds are hashed like any other content (the `directory` integrity +The two content kinds are hashed like any other content (the `directory` integrity is the sha512 of its JSON text), so the lockfile attests them and a frozen run -verifies them. `--fs` (either mode) requires an active bundle mode (`add`, `replace`, or `load`). +verifies them; a stat record has no content to hash — the `formats` entry itself is +the attestation, cross-checked bundle-vs-lockfile like every other format. `--fs` +(either mode) requires an active bundle mode (`add`, `replace`, or `load`). When a bundler plugin runs with its own `bundleFile` (a *sidecar*), `--fs` coordinates with it so the bundler's module graph isn't duplicated into the main bundle: a read the sidecar diff --git a/stasis-core/src/fs.js b/stasis-core/src/fs.js index e0a4021..049671f 100644 --- a/stasis-core/src/fs.js +++ b/stasis-core/src/fs.js @@ -2,7 +2,8 @@ // the filesystem readers // fs.readFileSync, fs.readdirSync, fs.lstatSync, fs.statSync so that, alongside the // module graph the loader hooks already capture, a program's explicit file/directory -// reads are recorded into the bundle (bundle=add|replace) and served back from it +// reads -- and the KIND (file vs directory) of the paths it lstat/stats -- are +// recorded into the bundle (bundle=add|replace) and served back from it // (bundle=load). Alongside the readers, the existence/canonical-path PROBES existsSync / // accessSync / realpathSync are served (load mode) so a tool that checks a file before // reading it finds a bundle-only file -- see the existsSync/accessSync/realpathSync bullet. @@ -31,17 +32,27 @@ // - readFileSync: the optional second argument (an `encoding` string, or an // options object with `encoding`) is honoured; the raw bytes are what we // capture/serve and `encoding` is applied on the way out. -// - lstatSync / statSync: serve-only, bundle=load, single-argument form. For a -// recorded path they return a Stats-like object whose isFile()/isDirectory() come +// - lstatSync / statSync: single-argument form, captured AND served -- but ONLY the +// path's kind, i.e. the type getters isFile()/isDirectory(); full Stats fields are +// never attested. CAPTURE (bundle=add|replace): the real call runs first, so the +// caller always gets the genuine Stats (and genuine errors); the observed kind is +// then recorded as a PAYLOAD-FREE stat record ('stat:file'/'stat:directory' in +// `formats` -- no bytes, no hash), so a stat-ONLY path -- never read, readdir'd, +// or imported -- still answers at load. A path that is neither a regular file nor +// a directory (a symlink under lstat, a socket, a FIFO, ...) isn't modelled and +// records nothing. SERVE (bundle=load): for a recorded path they return a +// Stats-like object whose isFile()/isDirectory() come // from the bundle (the common existence check); every other member is the real // stat while the file is still on disk, and a benign synthetic default (0 / epoch // / file-or-dir mode) once it's gone -- so fs wrappers that read more than // isFile()/isDirectory() (graceful-fs reads stats.uid/gid) keep working under -// bundle=load instead of re-throwing ENOENT. Not patched in capture mode (the file -// is on disk there, so the real Stats is correct), except to report a skipped -// source-map sidecar as absent (next bullet). statSync vs lstatSync differ only in -// symlink following, which matters solely for that on-disk passthrough -- each falls -// back to its own real implementation. +// bundle=load instead of re-throwing ENOENT. A skipped +// source-map sidecar is reported absent in both modes (next bullet). statSync vs +// lstatSync differ only in symlink following, which matters for the real-call +// paths (passthrough, capture, present-on-disk serve) -- each uses its own real +// implementation, and a stat of an in-root symlink records the TARGET's kind +// under the requested path (keyed exactly like a byte read of that path would +// be) while an lstat of one records nothing. // - existsSync / accessSync / realpathSync (+ their async/promise forms under =async): // existence/canonical-path PROBES, serve-only and bundle=load only. A path the bundle // records answers existent / accessible (read-only: F_OK/R_OK, while W_OK/X_OK defer) / @@ -57,7 +68,9 @@ // module loader). Because these are serve-only (never captured), a file that is ONLY // existence-probed -- never read or imported -- isn't recorded, so its probe falls // through to disk on load; babel.config.js is fine because it is also require()d, hence -// recorded. realpathSync returns the real (symlink-resolved) path while the file is on +// recorded. (An lstatSync/statSync of a path, by contrast, DOES record its kind -- see +// the lstat/stat bullet above -- after which these probes answer from that stat record +// too, since they consult the same getFsStatFamily.) realpathSync returns the real (symlink-resolved) path while the file is on // disk, and falls back to the LEXICALLY-resolved request path once it's bundle-only (a // symlink in a surviving ancestor dir is not re-resolved -- moot for the absent-config // case, but a canonicalize-then-compare under a symlinked ancestor could diverge between @@ -392,11 +405,31 @@ export function installFsHooks({ async: patchAsync, getState, markAborted, isLoa return realReaddirSync(path, options) } + // Capture-side stat recording, shared by the sync/async lstat/stat wrappers: record + // the path's KIND (file vs directory) as a payload-free stat record (state.addFsStat), + // so the type getters -- lstatSync(x).isFile()/.isDirectory(), the common existence- + // and-kind check -- of a stat-ONLY path answer from the bundle at load. Only those two + // kinds are modelled (the serve shim's contract); a symlink (under lstat), socket, + // FIFO, or device records nothing. The caller has already received (or will receive) + // the REAL Stats -- recording never alters what the program sees. Guards mirror the + // byte readers: a path whose real location escapes the root (in-tree symlink pointing + // out) is not attested, a file whose bytes a write-mode sidecar already carries is + // skipped, and a recording conflict (a kind flip mid-run) taints the capture via + // markAborted so nothing inconsistent is written. + const captureStat = (state, abs, stats) => { + const isDir = stats.isDirectory() + if (!isDir && !stats.isFile()) return + const url = pathToFileURL(abs).toString() + if (!realContained(state.root, abs) || state.attestedBySidecar(url)) return + try { state.addFsStat(url, isDir ? 'directory' : 'file') } catch (err) { markAborted(err) } + } + fs.lstatSync = function lstatSync(path, options) { const state = getState() - // Serve-only, single-argument form. Capture mode is left to the real lstatSync - // (the file is on disk, so its Stats is correct and complete) -- the lone exception - // being a skipped source-map sidecar, reported absent in both modes. + // Single-argument form. Serve (bundle=load) the bundle-backed type getters for a + // recorded path; capture (bundle=add|replace) the observed kind via captureStat, + // handing the caller the REAL Stats (and real errors, e.g. ENOENT) untouched. + // A skipped source-map sidecar is reported absent in both modes. if (state && options == null) { const abs = toAbsPath(path) if (abs !== null && withinRoot(state.root, abs)) { @@ -404,6 +437,13 @@ export function installFsHooks({ async: patchAsync, getState, markAborted, isLoa if (state.config.loadBundle) { const kind = state.getFsStatFamily(pathToFileURL(abs).toString()) if (kind !== undefined) return bundleStats(realLstatSync, path, kind === 'directory') + } else if (state.config.writeBundle && !isLoadingModule()) { + // (Node's loader stats module paths through internal bindings, not this + // public API, but skip during a load window anyway, symmetric with + // readFileSync/readdirSync.) + const stats = realLstatSync(path) + captureStat(state, abs, stats) + return stats } } } @@ -413,7 +453,10 @@ export function installFsHooks({ async: patchAsync, getState, markAborted, isLoa // statSync mirrors lstatSync exactly; the only difference is symlink following, // which is moot for a bundle-served path (there is no link to follow once the // bytes come from the bundle) but matters for the passthrough/present-on-disk - // case, so it falls back to the real statSync rather than lstatSync. + // case AND for capture -- a statSync of an in-root symlink records the TARGET's + // kind (keyed at the requested path, like a byte read of it), while an lstatSync + // of it records nothing (a symlink isn't modelled) -- so each uses its own real + // implementation. fs.statSync = function statSync(path, options) { const state = getState() if (state && options == null) { @@ -423,6 +466,10 @@ export function installFsHooks({ async: patchAsync, getState, markAborted, isLoa if (state.config.loadBundle) { const kind = state.getFsStatFamily(pathToFileURL(abs).toString()) if (kind !== undefined) return bundleStats(realStatSync, path, kind === 'directory') + } else if (state.config.writeBundle && !isLoadingModule()) { + const stats = realStatSync(path) + captureStat(state, abs, stats) + return stats } } } @@ -597,8 +644,10 @@ export function installFsHooks({ async: patchAsync, getState, markAborted, isLoa return realReaddirP(path, options) } - // lstat/stat are serve-only (like their sync siblings); capture is left to the real - // call. bundleStats's field fallback is synchronous, so it uses the sync real stat. + // lstat/stat capture and serve exactly like their sync siblings: serve the + // bundle-backed type getters, or record the real call's observed kind via + // captureStat while handing the caller the real Stats. bundleStats's field + // fallback is synchronous, so it uses the sync real stat. fs.lstat = function lstat(path, options, callback) { const cb = typeof options === 'function' ? options : callback // "single-argument" form: fn(path, cb) OR fn(path, null/undefined, cb). graceful-fs @@ -610,6 +659,13 @@ export function installFsHooks({ async: patchAsync, getState, markAborted, isLoa if (t?.mode === 'serve') { const kind = t.state.getFsStatFamily(t.url) if (kind !== undefined) { queueMicrotask(() => cb(null, bundleStats(realLstatSync, path, kind === 'directory'))); return } + } else if (t?.mode === 'capture') { + realLstat(path, (err, stats) => { + if (err) return cb(err) // faithful error on the user's original path; nothing recorded + captureStat(t.state, t.abs, stats) + cb(null, stats) + }) + return } return realLstat(path, options, callback) } @@ -625,6 +681,13 @@ export function installFsHooks({ async: patchAsync, getState, markAborted, isLoa if (t?.mode === 'serve') { const kind = t.state.getFsStatFamily(t.url) if (kind !== undefined) { queueMicrotask(() => cb(null, bundleStats(realStatSync, path, kind === 'directory'))); return } + } else if (t?.mode === 'capture') { + realStat(path, (err, stats) => { + if (err) return cb(err) + captureStat(t.state, t.abs, stats) + cb(null, stats) + }) + return } return realStat(path, options, callback) } @@ -635,6 +698,10 @@ export function installFsHooks({ async: patchAsync, getState, markAborted, isLoa if (t?.mode === 'serve') { const kind = t.state.getFsStatFamily(t.url) if (kind !== undefined) return bundleStats(realLstatSync, path, kind === 'directory') + } else if (t?.mode === 'capture') { + const stats = await realLstatP(path) + captureStat(t.state, t.abs, stats) + return stats } return realLstatP(path, options) } @@ -645,6 +712,10 @@ export function installFsHooks({ async: patchAsync, getState, markAborted, isLoa if (t?.mode === 'serve') { const kind = t.state.getFsStatFamily(t.url) if (kind !== undefined) return bundleStats(realStatSync, path, kind === 'directory') + } else if (t?.mode === 'capture') { + const stats = await realStatP(path) + captureStat(t.state, t.abs, stats) + return stats } return realStatP(path, options) } diff --git a/stasis-core/src/hooks.js b/stasis-core/src/hooks.js index 9743707..b597970 100644 --- a/stasis-core/src/hooks.js +++ b/stasis-core/src/hooks.js @@ -33,6 +33,7 @@ const NODEJS_FORMATS = new Set(['module', 'commonjs', 'json', 'module-typescript // the allowlist stays the single trust gate and the message is just UX. const NON_NODE_SOURCE_LANGUAGES = new Set(['solidity', 'php', 'bash', 'rust']) const RESOURCE_FORMATS = new Set(['resource', 'resource:base64']) +const STAT_FORMATS = new Set(['stat:file', 'stat:directory']) // The executable CommonJS formats (plain CJS + TS-CJS). Named like the sets above so // "which formats are CJS" lives in one place; the require-repair below keys off it. const CJS_FORMATS = new Set(['commonjs', 'commonjs-typescript']) @@ -56,6 +57,13 @@ function refuseNonNodeFormat(format, url) { `fs.readdirSync result is not executable JavaScript. Read it with fs.readdirSync instead (${url})` ) } + if (STAT_FORMATS.has(format)) { + throw new Error( + `[stasis] cannot import '${url}': the bundle carries only a payload-free stat record ` + + `('${format}', an fs.lstatSync/statSync capture attesting existence and kind) -- ` + + `not the file's content. Capture a run that reads or imports it to bundle its bytes.` + ) + } // Unknown format string: usually a tampered or forward-incompatible bundle. throw new Error( `[stasis] cannot execute '${url}': attested format '${format}' is not a Node loader ` + @@ -778,8 +786,9 @@ function patchCjsResolution() { // path.resolve()d form, matching native's normalized cache-key contract). Falling // through to native would stat a file the bundle carries but disk (pruned / // never-shipped node_modules) may not -- MODULE_NOT_FOUND on a servable path. - // getFsStat === 'file' (not mere presence) keeps a --fs-captured directory - // LISTING from hijacking native directory->main/index resolution. This widens + // hasFsFileContent (byte content, not getFsStat's kind) keeps a --fs-captured + // directory LISTING -- and a payload-free 'stat:*' record, whose bytes the + // bundle can NOT serve -- from hijacking native disk resolution. This widens // parentless reachability to any attested in-scope file when disk lacks the path; // that is bounded by attestation and the load hook's gates (getFile + // NODEJS_FORMATS), which remain the load-bearing checks. @@ -788,7 +797,7 @@ function patchCjsResolution() { // re-resolves through here, and the no-parent shape survives on >=24.18 too (the // deferred-resolution branch and createCJSNoSourceModuleWrap still take it). const url = pathToFileURL(request).toString() - if ((state.config.full || state.inNodeModules(url)) && state.getFsStat(url) === 'file') { + if ((state.config.full || state.inNodeModules(url)) && state.hasFsFileContent(url)) { return resolvePath(request) } } diff --git a/stasis-core/src/state.js b/stasis-core/src/state.js index ae8e704..8ffe0dc 100644 --- a/stasis-core/src/state.js +++ b/stasis-core/src/state.js @@ -11,7 +11,7 @@ import { Bundle } from './bundle.js' import { Lockfile } from './lockfile.js' import { canonicalizePath, sha512integrity, readFileSyncMaybe, noupsert } from './state-util.js' import { brotliOptions } from './brotli.js' -import { CODE_EXTENSIONS, classifyExtension, fileMapToObject, moduleFileKey, objectToMaps, sortPaths, splitNodeModulesPath } from './util.js' +import { CODE_EXTENSIONS, classifyExtension, fileMapToObject, isStatFormat, moduleFileKey, objectToMaps, sortPaths, splitNodeModulesPath } from './util.js' import corePackage from './package.cjs' // Object-destructure off the namespace rather than `import { ... } from 'node:fs'`: @@ -20,7 +20,7 @@ import corePackage from './package.cjs' // module.syncBuiltinESMExports() to refresh node:fs's ESM wrapper for user code. // Direct ESM destructured imports are live bindings and would re-resolve to the // mocked names when state.write() fires on beforeExit. -const { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } = fs +const { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } = fs const FILE_CONFIG = 'stasis.config.json' const FILE_LOCK = 'stasis.lock.json' @@ -1126,6 +1126,14 @@ export class State { if (!asResource && Bundle.isResourceFormat(format)) { throw new Error(`addFile: format '${format}' requires resource: true`) } + // A payload-free stat record is never a legal addFile format: addFile records + // CONTENT (bytes + hash), and a content record tagged 'stat:*' would desync the + // "stat records carry no payload" invariant every consumer relies on. The only + // producer is addFsStat; a caller passing it here (e.g. a forged child shard + // replayed by mergeShard) is refused at the recording site. + if (isStatFormat(format)) { + throw new Error(`addFile: format '${format}' is a payload-free stat record; use addFsStat`) + } // A resource can't be an entry: entries name code loaded by Node (the module // graph's roots), and Bundle.parse / assertEntry validate them as code-shaped // paths. A resource entry would land in this.entries -> codeBundle.entries @@ -1139,6 +1147,14 @@ export class State { // Canonicalize + bucket by the owning package (shared with addFsDir). const { absolute, file, dir, module, closestType } = this.#locateModule(url) + // A payload-free stat record (addFsStat: the path was lstat/stat'd before it was + // read or imported) is superseded by real content: drop it so the format + // inference/noupsert below records this file's ACTUAL format instead of + // conflicting with the weak 'stat:*' tag. The lockfile-baseline counterpart (a + // prior capture attested the path stat-only, this run reads it) lives in the + // flip-check relaxation below and #mergedFormats' upgrade rule. + if (isStatFormat(this.formats.get(file))) this.formats.delete(file) + // A code file can NEVER be recorded as a resource. `resources` -- the only gate // that admits an asset payload -- rejects code extensions outright (see // parseResourcesOption), so a resource:true call for a code extension means a @@ -1280,7 +1296,10 @@ export class State { // --lock=replace / --bundle=replace.) if (this.config.writeLockfile && this.#lockFormats !== null && format != null && inAttestedZone) { const attested = this.#lockFormats.get(file) - assert.ok(attested === undefined || attested === format, + // A payload-free 'stat:*' baseline is not a flip: the prior capture attested only + // the path's existence/kind, and this run observing real content UPGRADES the + // record (#mergedFormats overwrites the stat record with the real format). + assert.ok(attested === undefined || attested === format || isStatFormat(attested), `format flip for ${file}: lockfile attests '${attested}', observed '${format}' (bytes unchanged) -- re-run with --lock=replace to re-attest`) } @@ -1348,6 +1367,10 @@ export class State { assert.ok(Array.isArray(names) && names.every((n) => typeof n === 'string'), 'addFsDir requires an array of string names') const { file, dir, module } = this.#locateModule(url, { directory: true }) + // As in addFile: a payload-free stat record (the dir was lstat/stat'd before it + // was readdir'd) is superseded by the real listing -- drop it before the format + // noupsert below records 'directory'. + if (isStatFormat(this.formats.get(file))) this.formats.delete(file) const content = JSON.stringify(names.toSorted()) const format = 'directory' const integrity = sha512integrity(content) @@ -1382,6 +1405,35 @@ export class State { noupsert(this.formats, file, format) } + // Record an `fs.lstatSync(path)`/`fs.statSync(path)` capture (`stasis run --fs`, + // capture side, single-arg form): a PAYLOAD-FREE stat record -- `formats[file] = + // 'stat:file' | 'stat:directory'`, no bytes, no hash, no module-files entry -- + // attesting only that the path existed with that KIND. That is exactly what the + // load-side type-getter shim needs (bundleStats sources isFile()/isDirectory() + // from getFsStat), so a stat-ONLY path -- never read, readdir'd, or imported -- + // still answers `lstatSync(x).isDirectory()` (and, transitively, the + // existsSync/accessSync/realpathSync probes, which consult getFsStatFamily) at + // bundle=load once it's gone from disk. + // + // Skipped when the path already has a content record (bytes or a directory + // listing, whether captured this run or seeded from the lockfile/bundle): that + // record already answers getFsStat, and a weak stat record must never sit beside + // it (check-then-read -- lstatSync before readFileSync -- is the common pattern). + // The REVERSE order (stat first, content later) is handled by addFile/addFsDir + // dropping the stale stat record in place and by #mergedFormats' upgrade rule. A + // kind flip WITHIN the surviving stat records (stat'd as a file, later stat'd as + // a directory) conflicts at the noupsert and taints the capture, exactly like a + // mid-run byte conflict. + addFsStat(url, kind) { + assert.ok(kind === 'file' || kind === 'directory', `addFsStat: unsupported kind '${kind}'`) + const file = this.#canonicalFile(url) + if (this.hashes.has(file) || this.sources.has(file) || this.resources.has(file)) return + noupsert(this.formats, file, `stat:${kind}`) + // Forwarded to the root via shardSnapshot's #observed-filtered formats, like + // every other capture; only a child's shardSnapshot reads it, skip otherwise. + if (this.config.childProcess) this.#observed.add(file) + } + // Serve an `fs.readFileSync` from the bundle (`stasis run --fs`, bundle=load). // Returns the raw bytes as a Buffer, or undefined when the path was not captured // (the hook then falls back to a real disk read) or names a captured directory (a @@ -1427,26 +1479,53 @@ export class State { } for (const key of this.sources.keys()) add(key) for (const key of this.resources.keys()) add(key) + // Payload-free stat records prove existence too: a 'stat:file' at + // node_modules/dep/index.js implies node_modules and node_modules/dep are + // directories exactly like a content record would, and a 'stat:directory' + // likewise implies its own ancestors (the recorded dir itself is answered + // directly by getFsStat's format branch, not via this index). + for (const [key, format] of this.formats) if (isStatFormat(format)) add(key) this.#impliedDirIndex = dirs return dirs } // Classify a captured path for the `fs.lstatSync(x)/statSync(x)` .isFile()/.isDirectory() // shim (`stasis run --fs`, bundle=load): 'directory' for a readdir capture, 'file' for - // a readFileSync capture, 'directory' for an ancestor directory implied by a recorded - // file (e.g. `node_modules` when the bundle carries `node_modules/dep/index.js`), or - // undefined when the path wasn't captured (the shim then falls back to a real on-disk - // lstatSync). A directory lives in this.resources too, so the 'directory' format is - // checked first; implied dirs are checked last so an explicit file/dir record wins. + // a readFileSync capture, the recorded kind for a payload-free stat record + // ('stat:file'/'stat:directory', an lstat/stat capture of a path never read), + // 'directory' for an ancestor directory implied by a recorded file (e.g. + // `node_modules` when the bundle carries `node_modules/dep/index.js`), or + // undefined when the path wasn't captured (the shim then falls back to a real + // on-disk lstatSync). A directory lives in this.resources too, so the 'directory' + // format is checked first; content records are checked before stat records + // (defensive -- addFsStat never records beside content) and implied dirs come + // last so an explicit record always wins. NB: 'file' here means "recorded as a + // file", NOT "the bundle can serve its bytes" -- callers needing the latter + // (resolution shims, byte-serving skips) must use hasFsFileContent. getFsStat(url) { let file try { file = this.#canonicalFile(url) } catch { return undefined } - if (this.formats.get(file) === 'directory') return 'directory' + const format = this.formats.get(file) + if (format === 'directory') return 'directory' if (this.sources.has(file) || this.resources.has(file)) return 'file' + if (format === 'stat:file') return 'file' + if (format === 'stat:directory') return 'directory' if (this.#impliedDirs().has(file)) return 'directory' return undefined } + // True when the bundle carries actual BYTE content for `url` as a file -- code or + // a resource payload -- as opposed to a directory listing, a payload-free 'stat:*' + // existence record, or nothing. The CJS-resolution shim (hooks.js) and the + // sidecar capture-skip route "can the bundle serve this file's bytes?" through + // this instead of getFsStat, which also answers 'file' for stat-only records. + hasFsFileContent(url) { + let file + try { file = this.#canonicalFile(url) } catch { return false } + if (this.formats.get(file) === 'directory') return false + return this.sources.has(file) || this.resources.has(file) + } + // --fs <-> bundler-plugin coordination. A StasisWebpack/StasisEsbuild/StasisMetro SIDECAR // (its own bundleFile) captures the bundler's MODULE GRAPH into its OWN bundle. But the // bundler also reads every module's source through node:fs (webpack's inputFileSystem is @@ -1468,11 +1547,11 @@ export class State { // on purpose: a file only this State has stays subject to --fs's own dedup / byte-conflict // detection (addFsFile's noupsert); the skip applies solely to a file a *sidecar* carries. attestedBySidecar(url) { - // `=== 'file'` (not merely `!== undefined`): the skip targets a FILE a sidecar carries (the - // graph the bundler reads through node:fs). getFsStat also returns 'directory' for an implied - // ancestor of a carried file; matching that would over-skip. Inert today (a readFileSync of a - // directory throws EISDIR before this is consulted) but `=== 'file'` states the intent exactly. - for (const s of this.#sidecars) if (s.getFsStat(url) === 'file') return true + // hasFsFileContent (not getFsStat): the skip targets a file whose BYTES a sidecar carries + // (the graph the bundler reads through node:fs). getFsStat would also match a 'directory' + // implied ancestor of a carried file, or a payload-free 'stat:*' record -- neither of which + // carries bytes, so matching them would over-skip and leave a genuine read unattested. + for (const s of this.#sidecars) if (s.hasFsFileContent(url)) return true return false } @@ -1738,12 +1817,26 @@ export class State { // Union of lockfile-attested and observed/bundle-served formats, same // append-only stance as #mergedImports: conflicting formats for one file are // fatal (noupsert). Sidecars' formats are unioned in for the same reason. + // The one exception is a payload-free stat record ('stat:*', addFsStat): it is + // WEAK by design -- real content observed for the same file (a lock=add re-run + // READING a path a prior capture only stat'd, or a sidecar bundling a file this + // run also stat'd) UPGRADES the record instead of conflicting, and a stat record + // never displaces a real format. Two REAL formats -- and two divergent stat + // kinds (stat:file vs stat:directory) -- stay fatal. #mergedFormats() { const merged = new Map() + const mergeFormat = (file, format) => { + const existing = merged.get(file) + if (existing !== undefined && existing !== format) { + if (isStatFormat(existing) && !isStatFormat(format)) { merged.set(file, format); return } + if (isStatFormat(format) && !isStatFormat(existing)) return + } + noupsert(merged, file, format) + } if (this.#lockFormats) for (const [file, format] of this.#lockFormats) merged.set(file, format) - for (const [file, format] of this.formats) noupsert(merged, file, format) + for (const [file, format] of this.formats) mergeFormat(file, format) for (const sidecar of this.#sidecars) { - for (const [file, format] of sidecar.formats) noupsert(merged, file, format) + for (const [file, format] of sidecar.formats) mergeFormat(file, format) } return merged } @@ -1830,6 +1923,28 @@ export class State { return consumers > 1 ? reason : undefined } + // The formats map a bundle artifact declares. Normally `this.formats` as-is; the + // one adjustment is dropping a payload-free stat record ('stat:*') that the run's + // UNIFIED attestation upgraded to a real format -- a write-mode sidecar bundling a + // file this run also stat'd (stat-before-plugin-capture ordering, so addFile's + // in-place drop never saw it). The lockfile then attests the REAL format + // (#mergedFormats' upgrade), and a bundle still declaring 'stat:*' for that file + // would fail its lockfile cross-check at load (#mergeBundleMetadata). Cheap in the + // common case: with no write-mode sidecars the live map is already consistent + // (addFile/addFsDir drop superseded stat records in place) and is returned as-is. + #formatsForBundle() { + if (this.#sidecars.size === 0) return this.formats + const merged = this.#mergedFormats() + let out = null // copy-on-write: clone only when an entry must be dropped + for (const [file, format] of this.formats) { + if (isStatFormat(format) && merged.get(file) !== format) { + if (out === null) out = new Map(this.formats) + out.delete(file) + } + } + return out ?? this.formats + } + get sourceBundle() { // One bundle holds both code and resources; merge the two content maps (their key // sets are disjoint -- a file is either code or a resource) and let `formats` tag @@ -1847,7 +1962,7 @@ export class State { config: this.config.values, entries: this.entries, modules: this.#bundleModules(contents), - formats: this.formats, + formats: this.#formatsForBundle(), imports: this.imports, reason: this.#bundleReason(contents.keys()), }) @@ -1868,8 +1983,11 @@ export class State { // Resource files don't appear in `this.sources` (addFile routes by format), so // restricting #bundleModules to `this.sources` automatically yields the code-only // view. Formats and modules are filtered to drop resource-only dirs/files. + // Payload-free stat records are NOT resource formats, so they ride the code half + // (like the other content-less metadata, entries/imports) -- the resources half's + // strict resource-formats-only shape check would reject them. const codeFormats = new Map() - for (const [file, format] of this.formats) { + for (const [file, format] of this.#formatsForBundle()) { if (!Bundle.isResourceFormat(format)) codeFormats.set(file, format) } return new Bundle({ @@ -2151,6 +2269,30 @@ export class State { } } } + // Payload-free stat records (`--fs` lstat/stat captures) live ONLY in the shard's + // formats map -- no module-files entry -- so the modules loop above never visits + // them. Replay each by re-deriving the kind from DISK (the same disk-is-truth + // stance as the byte/listing re-reads: a shard can't inject a forged kind), via + // addFsStat so a path the root already attests with content is skipped, exactly + // as if this process had stat'd it. Range-check the path BEFORE touching disk, + // like the directory branch above; a path gone from disk (or of an unmodelled + // kind, e.g. now a symlink) is skipped, best-effort like every other replay. + if (lf.formats) { + for (const [file, format] of lf.formats) { + if (!isStatFormat(format)) continue + const absolute = resolve(this.root, file) + const relFromRoot = relative(this.root, absolute) + if (relFromRoot.startsWith('..') || isAbsolute(relFromRoot)) continue + try { + const stats = lstatSync(absolute) + const kind = stats.isDirectory() ? 'directory' : stats.isFile() ? 'file' : null + if (kind !== null) this.addFsStat(pathToFileURL(absolute).toString(), kind) + } catch { + // Gone from disk, or rejected (kind conflict with the root's own capture): + // skip -- the root's capture stays intact. + } + } + } if (lf.imports) { for (const [conditions, byParent] of lf.imports) { // Reverse #conditionsKey: '*' stays '*'; otherwise the key is a ', '-joined condition diff --git a/stasis-core/src/util.js b/stasis-core/src/util.js index 682a2d4..9512867 100644 --- a/stasis-core/src/util.js +++ b/stasis-core/src/util.js @@ -19,7 +19,11 @@ const sep = '/' // Source languages (analysis-only, not runnable by Node) — solidity, php, bash, rust // Resources (asset payloads) — resource (raw UTF-8), resource:base64 (binary) // Filesystem captures (`stasis run --fs`) — directory (a JSON-serialized, -// sorted `fs.readdirSync` listing; a resource-like raw-UTF-8 payload) +// sorted `fs.readdirSync` listing; a resource-like raw-UTF-8 payload), and the +// payload-free stat records stat:file / stat:directory (an `fs.lstatSync`/ +// `fs.statSync` capture: attests only that the path existed with that KIND — +// no bytes, no hash, no module-files entry — so isFile()/isDirectory() answer +// from the bundle at load; see isStatFormat below) // Adding a new format is a deliberate schema change: list it here AND extend // hooks.js's executable allowlist if Node should serve it from a bundle. export const KNOWN_FORMATS = new Set([ @@ -35,6 +39,8 @@ export const KNOWN_FORMATS = new Set([ 'resource', 'resource:base64', 'directory', + 'stat:file', + 'stat:directory', ]) // File extensions whose bytes are JS-shaped code -- Node's loader and the @@ -50,6 +56,14 @@ export const KNOWN_FORMATS = new Set([ // own set, so an fs-read tagged a `.jsx` 'resource' while the import tagged it code. export const CODE_EXTENSIONS = new Set(['js', 'mjs', 'cjs', 'ts', 'jsx', 'tsx', 'json', 'mts', 'cts']) +// The payload-free stat-record formats (`stasis run --fs` lstatSync/statSync captures). +// A stat record attests a path's KIND (file vs directory) without any content, so it +// lives ONLY in the `formats` maps — never in a module's hashed `files`, never as a +// bundle payload. It is deliberately WEAK: real content recorded for the same path (a +// read, a readdir listing, an import) supersedes it — see State's addFile/addFsDir +// stale-record drop and #mergedFormats' upgrade rule. +export const isStatFormat = (format) => format === 'stat:file' || format === 'stat:directory' + // Lowercased, dot-less extension of a path, or '' if none. export function pathExt(filePath) { const m = /\.([^./\\]+)$/.exec(filePath) diff --git a/stasis/src/cmd/build.js b/stasis/src/cmd/build.js index 1b9c47f..36b9390 100644 --- a/stasis/src/cmd/build.js +++ b/stasis/src/cmd/build.js @@ -21,13 +21,14 @@ export { bundleFromLockfile } const JS_EXTS = new Set(['.js', '.cjs', '.mjs', '.ts', '.cts', '.mts', '.jsx', '.tsx']) // Allowlist (NOT a blocklist) of per-file formats `stasis build` accepts in an artifact: -// the JS/TS/JSON code formats esbuild builds, plus the non-code payload formats -// (resource/directory) that are inert unless the SELECTED entry imports them. An allowlist +// the JS/TS/JSON code formats esbuild builds, plus the non-code formats -- resource/ +// directory payloads and the payload-free stat records (`--fs` lstat/stat captures) -- +// that are inert unless the SELECTED entry imports them. An allowlist // so a new language format added to `stasis bundle` later (e.g. a future `go`) is rejected // here by default, rather than slipping through to a cryptic esbuild failure in the build. const BUILDABLE_FORMATS = new Set([ 'module', 'commonjs', 'json', 'module-typescript', 'commonjs-typescript', - 'resource', 'resource:base64', 'directory', + 'resource', 'resource:base64', 'directory', 'stat:file', 'stat:directory', ]) // esbuild is an OPTIONAL peer dependency: `stasis build` is the only command that diff --git a/tests/cli-fs.test.js b/tests/cli-fs.test.js index 5e1e988..058ba2f 100644 --- a/tests/cli-fs.test.js +++ b/tests/cli-fs.test.js @@ -1079,3 +1079,235 @@ test('"fs" in stasis.config.json requires an active bundle mode', withTmp((t, tm t.assert.notEqual(r.status, 0) t.assert.match(r.stderr, /fs requires bundle=\(add\|replace\|load\)/) })) + +// ----- stat-ONLY paths are RECORDED (payload-free stat records) ------------------------ +// lstatSync/statSync used to be serve-only: a path that was only stat'd -- never read, +// readdir'd, or imported -- was not in the bundle, so on load its stat fell through to +// the real fs and threw ENOENT once the path was gone from disk. Capture now records the +// observed KIND as a payload-free 'stat:file'/'stat:directory' formats entry (no bytes, +// no hash), which the load-side type-getter shim (and the existence probes) answer from. + +test('run --fs records a stat-only path and serves its type getters on load', withTmp((t, tmp) => { + // .dat is deliberately NOT in the resources allowlist: a READ of it would abort the + // capture (undeclared extension), but a stat is extension-agnostic -- only the kind is + // recorded, no bytes -- so it must capture cleanly. + mkdirSync(join(tmp, 'src', 'zone', 'deep'), { recursive: true }) + writeFileSync(join(tmp, 'src', 'zone', 'deep', 'probe.dat'), 'PAYLOAD') + writeFileSync(join(tmp, 'src', 'entry.js'), [ + "import { lstatSync, statSync, existsSync } from 'node:fs'", + "import { join } from 'node:path'", + 'const zone = join(import.meta.dirname, "zone")', + 'const file = join(zone, "deep", "probe.dat")', + 'console.log(`file:${lstatSync(file).isFile()}:${lstatSync(file).isDirectory()}`)', + 'console.log(`dir:${statSync(zone).isDirectory()}`)', + 'console.log(`exists:${existsSync(file)}`)', // the probe answers from the stat record on load + // src/zone/deep is IMPLIED by the stat record at src/zone/deep/probe.dat -- never + // stat'd at capture (env-gated so the recorded entry is identical), so at load this + // exercises the implied-ancestor path for stat records specifically. + 'if (process.env.STASIS_TEST_IMPLIED) console.log(`implied:${statSync(join(zone, "deep")).isDirectory()}`)', + 'try { lstatSync(join(zone, "missing.dat")) } catch (e) { console.log(`miss:${e.code}`) }', + '', + ].join('\n')) + const bundlePath = join(tmp, 'snapshot.br') + const save = run(['run', '--lock=add', '--bundle=add', `--bundle-file=${bundlePath}`, '--fs=sync', 'src/entry.js'], { cwd: tmp }) + t.assert.equal(save.status, 0, `save stderr: ${save.stderr}`) + t.assert.equal(save.stdout, 'file:true:false\ndir:true\nexists:true\nmiss:ENOENT\n') + + // The records are PAYLOAD-FREE: a formats tag only, no bytes in the bundle, no hash in + // the lockfile's file lists. + const bundle = decode(bundlePath) + t.assert.equal(bundle.formats['src/zone/deep/probe.dat'], 'stat:file') + t.assert.equal(bundle.formats['src/zone'], 'stat:directory') + t.assert.equal(bundle.sources['.'].files['src/zone/deep/probe.dat'], undefined, 'no bytes recorded for a stat-only file') + t.assert.equal(bundle.sources['.'].files['src/zone'], undefined) + const lock = JSON.parse(readFileSync(join(tmp, 'stasis.lock.json'), 'utf-8')) + t.assert.equal(lock.formats['src/zone/deep/probe.dat'], 'stat:file') + t.assert.equal(lock.formats['src/zone'], 'stat:directory') + t.assert.equal(lock.sources['.'].files['src/zone/deep/probe.dat'], undefined, 'no hash for a payload-free record') + + rmSync(join(tmp, 'src', 'zone'), { recursive: true }) // gone from disk; only the stat records answer + const load = run(['run', '--lock=frozen', '--bundle=load', `--bundle-file=${bundlePath}`, '--fs=sync', 'src/entry.js'], + { cwd: tmp, env: { ...cleanEnv, STASIS_TEST_IMPLIED: '1' } }) + t.assert.equal(load.status, 0, `load stderr: ${load.stderr}`) + t.assert.equal(load.stdout, 'file:true:false\ndir:true\nexists:true\nimplied:true\nmiss:ENOENT\n') +})) + +test('run --fs collapses a path that is both stat\'d and read into the content record (either order)', withTmp((t, tmp) => { + // check-then-read (x) and read-then-check (y): the weak stat record must never conflict + // with -- or survive next to -- the real content record for the same path. + writeFileSync(join(tmp, 'src', 'x.txt'), 'X') + writeFileSync(join(tmp, 'src', 'y.txt'), 'Y') + writeFileSync(join(tmp, 'src', 'entry.js'), [ + "import { lstatSync, statSync, readFileSync } from 'node:fs'", + "import { join } from 'node:path'", + 'const x = join(import.meta.dirname, "x.txt")', + 'const y = join(import.meta.dirname, "y.txt")', + 'const okX = lstatSync(x).isFile()', // stat FIRST ... + 'const dataX = readFileSync(x, "utf8")', // ... then read + 'const dataY = readFileSync(y, "utf8")', // read FIRST ... + 'const okY = statSync(y).isFile()', // ... then stat + 'console.log(`x:${okX}:${dataX} y:${okY}:${dataY}`)', + '', + ].join('\n')) + const bundlePath = join(tmp, 'snapshot.br') + const save = run(['run', '--lock=add', '--bundle=add', `--bundle-file=${bundlePath}`, '--fs=sync', 'src/entry.js'], { cwd: tmp }) + t.assert.equal(save.status, 0, `save stderr: ${save.stderr}`) + t.assert.equal(save.stdout, 'x:true:X y:true:Y\n') + t.assert.doesNotMatch(save.stderr, /capture aborted/, 'stat + read of one path must not conflict') + + const bundle = decode(bundlePath) + t.assert.equal(bundle.formats['src/x.txt'], 'resource', 'the content record wins over the stat record') + t.assert.equal(bundle.formats['src/y.txt'], 'resource') + t.assert.equal(bundle.sources['.'].files['src/x.txt'], 'X') + + rmSync(join(tmp, 'src', 'x.txt')) + rmSync(join(tmp, 'src', 'y.txt')) + const load = run(['run', '--lock=frozen', '--bundle=load', `--bundle-file=${bundlePath}`, '--fs=sync', 'src/entry.js'], { cwd: tmp }) + t.assert.equal(load.status, 0, `load stderr: ${load.stderr}`) + t.assert.equal(load.stdout, 'x:true:X y:true:Y\n') +})) + +test('run --fs lock=add re-run upgrades a stat-only record to a content record (no format-flip abort)', withTmp((t, tmp) => { + // Run A only stats x.txt -> lockfile+bundle attest 'stat:file'. Run B (lock=add, + // bundle=add against the same artifacts; identical entry, env-gated) also READS it: + // the weak stat baseline must upgrade to 'resource' instead of tripping the + // format-flip guard or the merged-formats conflict. + writeFileSync(join(tmp, 'src', 'x.txt'), 'X') + writeFileSync(join(tmp, 'src', 'entry.js'), [ + "import { lstatSync, readFileSync } from 'node:fs'", + "import { join } from 'node:path'", + 'const x = join(import.meta.dirname, "x.txt")', + 'let out = `stat:${lstatSync(x).isFile()}`', + 'if (process.env.STASIS_TEST_READ) out += `:${readFileSync(x, "utf8")}`', + 'console.log(out)', + '', + ].join('\n')) + const bundlePath = join(tmp, 'snapshot.br') + const runA = run(['run', '--lock=add', '--bundle=add', `--bundle-file=${bundlePath}`, '--fs=sync', 'src/entry.js'], { cwd: tmp }) + t.assert.equal(runA.status, 0, `run A stderr: ${runA.stderr}`) + t.assert.equal(decode(bundlePath).formats['src/x.txt'], 'stat:file') + t.assert.equal(JSON.parse(readFileSync(join(tmp, 'stasis.lock.json'), 'utf-8')).formats['src/x.txt'], 'stat:file') + + const runB = run(['run', '--lock=add', '--bundle=add', `--bundle-file=${bundlePath}`, '--fs=sync', 'src/entry.js'], + { cwd: tmp, env: { ...cleanEnv, STASIS_TEST_READ: '1' } }) + t.assert.equal(runB.status, 0, `run B stderr: ${runB.stderr}`) + t.assert.equal(runB.stdout, 'stat:true:X\n') + t.assert.doesNotMatch(runB.stderr, /format flip|capture aborted/, 'upgrading a stat-only baseline must not abort') + const upgraded = decode(bundlePath) + t.assert.equal(upgraded.formats['src/x.txt'], 'resource', 'the re-run upgraded the stat record to content') + t.assert.equal(upgraded.sources['.'].files['src/x.txt'], 'X') + t.assert.equal(JSON.parse(readFileSync(join(tmp, 'stasis.lock.json'), 'utf-8')).formats['src/x.txt'], 'resource') + + rmSync(join(tmp, 'src', 'x.txt')) + const load = run(['run', '--lock=frozen', '--bundle=load', `--bundle-file=${bundlePath}`, '--fs=sync', 'src/entry.js'], + { cwd: tmp, env: { ...cleanEnv, STASIS_TEST_READ: '1' } }) + t.assert.equal(load.status, 0, `load stderr: ${load.stderr}`) + t.assert.equal(load.stdout, 'stat:true:X\n') +})) + +test('run --fs=async records stat-only paths via callback lstat and fs.promises.stat', withTmp((t, tmp) => { + mkdirSync(join(tmp, 'src', 'd')) + writeFileSync(join(tmp, 'src', 'd', 'probe.dat'), 'P') + writeFileSync(join(tmp, 'src', 'entry.js'), [ + "import { lstat } from 'node:fs'", + "import { stat as statP } from 'node:fs/promises'", + "import { join } from 'node:path'", + 'const dir = join(import.meta.dirname, "d")', + 'const file = join(dir, "probe.dat")', + 'const p = (fn, ...a) => new Promise((res, rej) => fn(...a, (e, v) => (e ? rej(e) : res(v))))', + 'console.log(`cb-lstat-dir:${(await p(lstat, dir)).isDirectory()}`)', + 'console.log(`p-stat-file:${(await statP(file)).isFile()}`)', + '', + ].join('\n')) + const bundlePath = join(tmp, 'snapshot.br') + const save = run(['run', '--lock=add', '--bundle=add', `--bundle-file=${bundlePath}`, '--fs=async', 'src/entry.js'], { cwd: tmp }) + t.assert.equal(save.status, 0, `save stderr: ${save.stderr}`) + const bundle = decode(bundlePath) + t.assert.equal(bundle.formats['src/d'], 'stat:directory') + t.assert.equal(bundle.formats['src/d/probe.dat'], 'stat:file') + + rmSync(join(tmp, 'src', 'd'), { recursive: true }) + const load = run(['run', '--lock=frozen', '--bundle=load', `--bundle-file=${bundlePath}`, '--fs=async', 'src/entry.js'], { cwd: tmp }) + t.assert.equal(load.status, 0, `load stderr: ${load.stderr}`) + t.assert.equal(load.stdout, 'cb-lstat-dir:true\np-stat-file:true\n') +})) + +test('run --fs does NOT attest a stat of a symlink that escapes the root, nor a symlink itself', withTmp((t, tmp) => { + // Security stance mirrors the byte readers: a statSync THROUGH an in-tree symlink + // pointing outside the root is answered from disk but never attested. An lstatSync of + // an in-root symlink reports a symlink -- a kind the shim doesn't model -- and records + // nothing either (capture stays untainted in both cases). + const outside = mkdtempSync(join(tmpdir(), 'stasis-fs-stat-OUTSIDE-')) + try { + symlinkSync(outside, join(tmp, 'src', 'peek')) + writeFileSync(join(tmp, 'src', 'real.txt'), 'R') + symlinkSync(join(tmp, 'src', 'real.txt'), join(tmp, 'src', 'link.txt')) + writeFileSync(join(tmp, 'src', 'entry.js'), [ + "import { statSync, lstatSync } from 'node:fs'", + "import { join } from 'node:path'", + 'const d = import.meta.dirname', + 'console.log(`peek:${statSync(join(d, "peek")).isDirectory()}`)', // follows the escaping link + 'console.log(`link:${lstatSync(join(d, "link.txt")).isSymbolicLink()}`)', // a symlink kind + '', + ].join('\n')) + const bundlePath = join(tmp, 'snapshot.br') + const r = run(['run', '--lock=add', '--bundle=add', `--bundle-file=${bundlePath}`, '--fs=sync', 'src/entry.js'], { cwd: tmp }) + t.assert.equal(r.status, 0, `stderr: ${r.stderr}`) + t.assert.equal(r.stdout, 'peek:true\nlink:true\n') // the program still sees the real fs + t.assert.doesNotMatch(r.stderr, /capture aborted/) + const bundle = decode(bundlePath) + t.assert.equal(bundle.formats['src/peek'], undefined, 'escaping-symlink stat must not be attested') + t.assert.equal(bundle.formats['src/link.txt'], undefined, 'a symlink kind is not modelled, so not recorded') + } finally { + rmSync(outside, { recursive: true, force: true }) + } +})) + +test('run --fs --child-process forwards a child-only stat record through the shard channel', (t) => { + // The forked worker (and ONLY the worker -- the root never touches the path) lstats + // statonly.dat, a file nothing reads. Its payload-free stat record can reach the root's + // artifacts only via the child's shard: shardSnapshot forwards the formats entry, and + // mergeShard replays it by re-deriving the kind from disk. Then a load run -- with the + // probed file gone -- must serve the child's lstat from the bundle. + const forkShardFixture = join(dirname(fileURLToPath(import.meta.url)), 'fixtures', 'cli-run-fork-shard') + const tmp = mkdtempSync(join(tmpdir(), 'stasis-fs-shard-')) + try { + cpSync(forkShardFixture, tmp, { recursive: true }) + const bundlePath = join(tmp, 'snapshot.br') + const env = { ...cleanEnv, STAT_PROBE: '1' } + const save = run(['run', '--lock=add', '--bundle=add', `--bundle-file=${bundlePath}`, '--fs=sync', '--child-process', 'src/entry.js'], { cwd: tmp, env }) + t.assert.equal(save.status, 0, `save stderr: ${save.stderr}`) + t.assert.match(save.stdout, /WORKER stat-only=true/, 'the forked child stat-probed the file') + + const lock = JSON.parse(readFileSync(join(tmp, 'stasis.lock.json'), 'utf-8')) + t.assert.equal(lock.formats['src/statonly.dat'], 'stat:file', 'child-only stat record attested via the shard merge') + t.assert.equal(lock.sources['.'].files['src/statonly.dat'], undefined, 'payload-free: no hash entry') + t.assert.equal(decode(bundlePath).formats['src/statonly.dat'], 'stat:file') + + rmSync(join(tmp, 'src', 'statonly.dat')) // gone; only the bundle's stat record can answer the child's lstat + const load = run(['run', '--lock=frozen', '--bundle=load', `--bundle-file=${bundlePath}`, '--fs=sync', '--child-process', 'src/entry.js'], { cwd: tmp, env }) + t.assert.equal(load.status, 0, `load stderr: ${load.stderr}`) + t.assert.match(load.stdout, /WORKER stat-only=true/, "the child's lstat of the absent file served from the bundle") + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +}) + +test('run --fs only captures the single-argument lstatSync/statSync (options pass through)', withTmp((t, tmp) => { + // Same scope rule as readdirSync: an options form (bigint, throwIfNoEntry) is out of + // scope -- passed through untouched and NOT recorded. + writeFileSync(join(tmp, 'src', 'opt.dat'), 'O') + writeFileSync(join(tmp, 'src', 'entry.js'), [ + "import { lstatSync, statSync } from 'node:fs'", + "import { join } from 'node:path'", + 'const p = join(import.meta.dirname, "opt.dat")', + 'console.log(`big:${typeof lstatSync(p, { bigint: true }).size}`)', + 'console.log(`quiet:${statSync(join(import.meta.dirname, "nope.dat"), { throwIfNoEntry: false })}`)', + '', + ].join('\n')) + const bundlePath = join(tmp, 'snapshot.br') + const r = run(['run', '--lock=add', '--bundle=add', `--bundle-file=${bundlePath}`, '--fs=sync', 'src/entry.js'], { cwd: tmp }) + t.assert.equal(r.status, 0, `stderr: ${r.stderr}`) + t.assert.equal(r.stdout, 'big:bigint\nquiet:undefined\n') + t.assert.equal(decode(bundlePath).formats['src/opt.dat'], undefined, 'an options-form stat must not be captured') +})) diff --git a/tests/fixtures/cli-run-fork-shard/src/statonly.dat b/tests/fixtures/cli-run-fork-shard/src/statonly.dat new file mode 100644 index 0000000..d7c8474 --- /dev/null +++ b/tests/fixtures/cli-run-fork-shard/src/statonly.dat @@ -0,0 +1 @@ +stat-only payload \ No newline at end of file diff --git a/tests/fixtures/cli-run-fork-shard/src/worker.js b/tests/fixtures/cli-run-fork-shard/src/worker.js index d5755c8..85031bf 100644 --- a/tests/fixtures/cli-run-fork-shard/src/worker.js +++ b/tests/fixtures/cli-run-fork-shard/src/worker.js @@ -13,5 +13,16 @@ if (process.env.SPAWN_GRANDCHILD) { await new Promise((resolve) => gc.on('exit', resolve)) } +// Env-gated (test-only): lstat a path NOTHING in the tree ever reads -- a stat-ONLY +// observation unique to this forked child, so its payload-free stat record can reach +// the root's lockfile/bundle exclusively via the shard channel (mergeShard's stat +// replay). Requires an --fs run; inert otherwise. +if (process.env.STAT_PROBE) { + const { lstatSync } = await import('node:fs') + const { fileURLToPath } = await import('node:url') + const probed = lstatSync(fileURLToPath(new URL('./statonly.dat', import.meta.url))) + console.log(`WORKER stat-only=${probed.isFile()}`) +} + console.log(`WORKER extra=${extra}`) if (process.send) process.send('done') From 1e6e2ced5eac3ba793e0edfed5601f51af170899 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 17:42:00 +0000 Subject: [PATCH 2/2] fix(state): follow symlinks when replaying shard stat records mergeShard replayed a child's payload-free stat records with lstatSync, which reports the link itself for a path that is a symlink on disk -- but a child's statSync capture follows links and records the TARGET's kind keyed at the requested path (pnpm's public node_modules/ links are exactly this shape), so every such record was silently dropped at the merge and a later bundle=load run ENOENT'd on it. Replay with statSync instead: it reproduces the recorded kind for every key a legitimate shard can carry (an lstat of a symlink was never recorded at all, and for non-links lstat and stat agree). Because statSync follows links, re-assert the capture side's containment invariant (fs.js captureStat's realContained) before attesting: the real location must stay inside the root's real path, so a shard entry keyed at an in-root symlink pointing out of the tree is skipped, never recorded. A dangling link now falls into the existing gone-from-disk skip. Reported by the Copilot review on #66. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017z3Eb4Udqc5BkdQtt2YA72 --- stasis-core/src/state.js | 29 +++++++++++++++---- tests/cli-fs.test.js | 14 ++++++++- .../fixtures/cli-run-fork-shard/src/worker.js | 8 ++++- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/stasis-core/src/state.js b/stasis-core/src/state.js index 8ffe0dc..ae2d27a 100644 --- a/stasis-core/src/state.js +++ b/stasis-core/src/state.js @@ -20,7 +20,7 @@ import corePackage from './package.cjs' // module.syncBuiltinESMExports() to refresh node:fs's ESM wrapper for user code. // Direct ESM destructured imports are live bindings and would re-resolve to the // mocked names when state.write() fires on beforeExit. -const { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } = fs +const { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync } = fs const FILE_CONFIG = 'stasis.config.json' const FILE_LOCK = 'stasis.lock.json' @@ -2276,20 +2276,37 @@ export class State { // addFsStat so a path the root already attests with content is skipped, exactly // as if this process had stat'd it. Range-check the path BEFORE touching disk, // like the directory branch above; a path gone from disk (or of an unmodelled - // kind, e.g. now a symlink) is skipped, best-effort like every other replay. + // kind, e.g. a socket) is skipped, best-effort like every other replay. if (lf.formats) { + let realRoot // resolved once, on the first record: the containment anchor below for (const [file, format] of lf.formats) { if (!isStatFormat(format)) continue const absolute = resolve(this.root, file) const relFromRoot = relative(this.root, absolute) if (relFromRoot.startsWith('..') || isAbsolute(relFromRoot)) continue try { - const stats = lstatSync(absolute) + // statSync (follow symlinks), NOT lstatSync: the child's capture followed them + // -- a statSync of an in-root symlink records the TARGET's kind keyed at the + // requested path (pnpm's public node_modules/ links are exactly this + // shape), and an lstat of a symlink was never recorded at all -- so for every + // key a legitimate shard can carry, statSync reproduces the recorded kind + // while lstatSync would report the link itself and silently drop the record. + const stats = statSync(absolute) const kind = stats.isDirectory() ? 'directory' : stats.isFile() ? 'file' : null - if (kind !== null) this.addFsStat(pathToFileURL(absolute).toString(), kind) + if (kind === null) continue + // Because statSync follows links, re-assert the capture side's containment + // (fs.js captureStat's realContained) before attesting: the REAL location must + // stay inside the root's real path, so a shard entry keyed at an in-root + // symlink that points OUT of the tree is skipped, never recorded. + if (realRoot === undefined) { + try { realRoot = realpathSync(this.root) } catch { realRoot = this.root } + } + const relReal = relative(realRoot, realpathSync(absolute)) + if (relReal !== '' && (relReal.startsWith('..') || isAbsolute(relReal))) continue + this.addFsStat(pathToFileURL(absolute).toString(), kind) } catch { - // Gone from disk, or rejected (kind conflict with the root's own capture): - // skip -- the root's capture stays intact. + // Gone from disk (or a dangling link), or rejected (kind conflict with the + // root's own capture): skip -- the root's capture stays intact. } } } diff --git a/tests/cli-fs.test.js b/tests/cli-fs.test.js index 058ba2f..a97502b 100644 --- a/tests/cli-fs.test.js +++ b/tests/cli-fs.test.js @@ -1273,21 +1273,33 @@ test('run --fs --child-process forwards a child-only stat record through the sha const tmp = mkdtempSync(join(tmpdir(), 'stasis-fs-shard-')) try { cpSync(forkShardFixture, tmp, { recursive: true }) + // The worker also statSync's THROUGH this in-root symlink: the record lands at the + // LINK's path with the TARGET's kind (the pnpm node_modules/ shape), and the + // root's replay must follow the link too -- with lstat it would see a symlink and + // silently drop the record (the Copilot-reported regression). + symlinkSync('statonly.dat', join(tmp, 'src', 'statlink.dat')) const bundlePath = join(tmp, 'snapshot.br') const env = { ...cleanEnv, STAT_PROBE: '1' } const save = run(['run', '--lock=add', '--bundle=add', `--bundle-file=${bundlePath}`, '--fs=sync', '--child-process', 'src/entry.js'], { cwd: tmp, env }) t.assert.equal(save.status, 0, `save stderr: ${save.stderr}`) t.assert.match(save.stdout, /WORKER stat-only=true/, 'the forked child stat-probed the file') + t.assert.match(save.stdout, /WORKER stat-link=true/, 'the forked child stat-probed through the symlink') const lock = JSON.parse(readFileSync(join(tmp, 'stasis.lock.json'), 'utf-8')) t.assert.equal(lock.formats['src/statonly.dat'], 'stat:file', 'child-only stat record attested via the shard merge') + t.assert.equal(lock.formats['src/statlink.dat'], 'stat:file', 'a stat-through-symlink record survives the shard replay') t.assert.equal(lock.sources['.'].files['src/statonly.dat'], undefined, 'payload-free: no hash entry') t.assert.equal(decode(bundlePath).formats['src/statonly.dat'], 'stat:file') + t.assert.equal(decode(bundlePath).formats['src/statlink.dat'], 'stat:file') - rmSync(join(tmp, 'src', 'statonly.dat')) // gone; only the bundle's stat record can answer the child's lstat + // Both gone (the link now points at nothing before it's removed too); only the + // bundle's stat records can answer the child's probes. + rmSync(join(tmp, 'src', 'statonly.dat')) + rmSync(join(tmp, 'src', 'statlink.dat')) const load = run(['run', '--lock=frozen', '--bundle=load', `--bundle-file=${bundlePath}`, '--fs=sync', '--child-process', 'src/entry.js'], { cwd: tmp, env }) t.assert.equal(load.status, 0, `load stderr: ${load.stderr}`) t.assert.match(load.stdout, /WORKER stat-only=true/, "the child's lstat of the absent file served from the bundle") + t.assert.match(load.stdout, /WORKER stat-link=true/, "the child's stat of the absent symlink served from the bundle") } finally { rmSync(tmp, { recursive: true, force: true }) } diff --git a/tests/fixtures/cli-run-fork-shard/src/worker.js b/tests/fixtures/cli-run-fork-shard/src/worker.js index 85031bf..19dcf6e 100644 --- a/tests/fixtures/cli-run-fork-shard/src/worker.js +++ b/tests/fixtures/cli-run-fork-shard/src/worker.js @@ -18,10 +18,16 @@ if (process.env.SPAWN_GRANDCHILD) { // the root's lockfile/bundle exclusively via the shard channel (mergeShard's stat // replay). Requires an --fs run; inert otherwise. if (process.env.STAT_PROBE) { - const { lstatSync } = await import('node:fs') + const { lstatSync, statSync } = await import('node:fs') const { fileURLToPath } = await import('node:url') const probed = lstatSync(fileURLToPath(new URL('./statonly.dat', import.meta.url))) console.log(`WORKER stat-only=${probed.isFile()}`) + // A stat THROUGH an in-root symlink (statlink.dat -> statonly.dat, created by the + // test): statSync follows it, so the record carries the TARGET's kind keyed at the + // LINK's path. The root's shard replay must follow the link too (statSync, not + // lstatSync) or this record is silently dropped at the merge. + const linked = statSync(fileURLToPath(new URL('./statlink.dat', import.meta.url))) + console.log(`WORKER stat-link=${linked.isFile()}`) } console.log(`WORKER extra=${extra}`)