diff --git a/docs/content/limitations/native-worker-threads.mdx b/docs/content/limitations/native-worker-threads.mdx index 5bd259114b1..de01ac7d0da 100644 --- a/docs/content/limitations/native-worker-threads.mdx +++ b/docs/content/limitations/native-worker-threads.mdx @@ -1,5 +1,8 @@ `node:worker_threads` runs workers as native threads. Worker entry points must resolve at build time: use a source-relative `new URL("./worker.ts", import.meta.url)`, the current module URL, an absolute path, or CommonJS `__filename`. +A module that runs in a Node island starts Node's own Worker instead: its entry must be a source-relative `new URL("./worker.ts", import.meta.url)` built from that module's URL, the build ships the entry and the modules it imports beside the island's modules, and the worker thread evaluates its own copies of them. +The build refuses a Worker whose entry it cannot resolve to such a path, and a TypeScript module of the program that ships this way whose types Node cannot strip in a worker thread. +A TypeScript module the worker's entry imports from outside the program ships unchecked, and Node reports its own error there if it cannot strip it. Each worker initializes its own module state, event loop, environment snapshot, and reference-counted heap. Nested workers inherit their parent's environment snapshot. Supported constructor options are `workerData`, `argv`, a `transferList` of MessagePorts, `name` (reported by `threadName` and `worker.threadName`), `stdout`, `stderr` and `stdin`, and `resourceLimits.stackSizeMb`, which sets the worker thread's stack size (at least 192 KiB; without it a worker's stack is 8 MiB, where Node's is 4 MiB). diff --git a/packages/compiler/src/executable/prepare.ts b/packages/compiler/src/executable/prepare.ts index 9412fbcd739..c62c6ca50de 100644 --- a/packages/compiler/src/executable/prepare.ts +++ b/packages/compiler/src/executable/prepare.ts @@ -52,7 +52,7 @@ import { isIslandReasonCode, } from "../node-islands/classify.js"; import { packageDirOf, packageVersionOf } from "../node-islands/origin.js"; -import { describe } from "../node-islands/summary.js"; +import { describe, type NodeIslandReason } from "../node-islands/summary.js"; import { resolveBareModule } from "../frontend/resolve.js"; import type { NodeIslandScan } from "../node-islands/scan.js"; import { growIslandSet, type IslandGrowthContext } from "../node-islands/growth.js"; @@ -248,7 +248,13 @@ export function prepareExecutableModule( // Sloppy user modules the lowering refused join the same retry // (DESIGN.md note 1 §3.2): one reload with every island at once, // never one round per module. - const grownModules = loweringModules(entryPath, opts, lowered.diagnostics, lowerModules); + const grownModules = loweringModules( + entryPath, + opts, + lowered.diagnostics, + lowerModules, + (files) => fe.nodeIslandScan?.({ userJs: false, stripFor: files }) ?? null, + ); if (grown !== null || grownModules !== null) { fe.dispose(); frontendDisposed(); @@ -450,6 +456,14 @@ const SLOPPY_REASONS: ReadonlySet = new Set([ * builtin, so it runs in Node, where an island consumer sees it. */ const PATCH_REASONS: ReadonlySet = new Set(["builtin-patch"]); +/** The runtime reasons a TypeScript module turns into an island module for: + * the JavaScript runtime owns the construct's base class, so the module + * needs nothing shipped beside it. A load only the runtime can resolve (a + * computed specifier, a worker entry path) stays refused — the shipped tree + * cannot carry what the module loads, and an island would fail where the + * native build ran. */ +const ISLAND_RUN_REASONS: ReadonlySet = new Set(["host-class-base"]); + /** The program modules that run in Node islands, by file: modules marked * with `--node-island=` or `scriptc.nodeIslands.modules`, and (under * `--node-islands=auto`) user JavaScript the pre-scan found a sloppy-only @@ -1055,28 +1069,59 @@ function loweringModules( opts: CompileRequestOptions, diagnostics: readonly ScrDiagnostic[], known: ReadonlyMap, + scanForRun: (files: readonly string[]) => NodeIslandScan | null, ): Map | null { const policy = nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands); if (policy.mode !== "auto" || known.size > 16) return null; const grown = new Map(known); + // TypeScript modules the refusals name, with the reason that names them + // (a module joins once, with its first reason). + const runtimeReasons = new Map(); for (const d of diagnostics) { if (!isIslandReasonCode(d.code)) continue; const file = d.loc.file; // Packages run in Node through their own decision // (node-islands/growth.ts). if (file.includes("/node_modules/") || packageDirOf(file) !== null) continue; - if (!isJsSourceFileName(file) || sameFile(file, entryPath)) continue; - if (grown.has(file)) continue; + if (sameFile(file, entryPath) || grown.has(file) || runtimeReasons.has(file)) continue; const reason = classifyDiagnostic(d, "refusal"); // A gap is a construct scriptc does not implement, not a sloppy // semantic: running it in Node would hide a compiler limitation. if (reason.class === "gap") continue; - grown.set(file, { - file, - class: reason.class, - reason: describe(reason), - decision: "auto", - }); + if (isJsSourceFileName(file)) { + grown.set(file, { + file, + class: reason.class, + reason: describe(reason), + decision: "auto", + }); + continue; + } + // A TypeScript module runs in Node only for a need of the JavaScript + // runtime itself that the shipped tree satisfies (ISLAND_RUN_REASONS): + // Node then does what the native pipeline could not. An object-model + // need is a JavaScript module's sloppy semantics, and a refusal caused + // by another declaration is that declaration's cascade. + if (!ISLAND_RUN_REASONS.has(reason.reason) || d.cause !== undefined) continue; + if (!/\.(?:ts|mts|cts)$/.test(file) || /\.d\.[cm]?ts$/.test(file)) continue; + runtimeReasons.set(file, reason); + } + if (runtimeReasons.size > 0) { + const scan = scanForRun([...runtimeReasons.keys()]); + for (const [file, reason] of runtimeReasons) { + const module = scan?.modules.find((m) => m.file === file); + // Node runs TypeScript by erasing types in place: a module that needs + // a transform, or resolves a relative import only without its + // extension, cannot run in an island as written, and its refusal + // stays the build's answer. + if (module === undefined || module.stripProblem !== null) continue; + grown.set(file, { + file, + class: reason.class, + reason: describe(reason), + decision: "auto", + }); + } } return grown.size === known.size ? null : grown; } diff --git a/packages/compiler/src/frontend/fork-target.ts b/packages/compiler/src/frontend/fork-target.ts index 938cb1b5c50..a5c8b977945 100644 --- a/packages/compiler/src/frontend/fork-target.ts +++ b/packages/compiler/src/frontend/fork-target.ts @@ -301,6 +301,57 @@ function staticString( return null; } +/** Whether an entry expression names its file relative to the module that + * starts it: `import.meta.url`, `__filename`, or a URL whose base is one of + * those (`new URL("./entry.ts", import.meta.url)`, through consts, + * templates and fileURLToPath). That relation is the one a shipped Node + * island tree keeps, because every collected file keeps its layout + * relative to one root; an absolute path or a `file:` URL built from + * anything else names the build machine, which the tree does not hold. */ +export function fromModuleUrl( + program: ts.Program, + expr: ts.Expression, + seen: Set = new Set(), +): boolean { + const current = strip(expr); + if (importMetaUrl(current)) return true; + if (ts.isIdentifier(current) && current.text === "__filename") + return isNodeGlobal(program, current); + const init = constInitializer(program, current); + if (init !== null && ts.isIdentifier(current)) { + const symbol = program.getTypeChecker().getSymbolAtLocation(current); + if (symbol === undefined || seen.has(symbol)) return false; + seen.add(symbol); + const value = fromModuleUrl(program, init, seen); + seen.delete(symbol); + return value; + } + if ( + ts.isCallExpression(current) && + !current.questionDotToken && + current.arguments.nodes.length === 1 && + ts.isIdentifier(current.expression) && + isBuiltinMemberImport(program, current.expression, "url", "fileURLToPath") + ) + return fromModuleUrl(program, current.arguments.nodes[0]!, seen); + if ( + ts.isNewExpression(current) && + !current.questionDotToken && + ts.isIdentifier(current.expression) && + (isBuiltinMemberImport(program, current.expression, "url", "URL") || + (current.expression.text === "URL" && isNodeGlobal(program, current.expression))) + ) { + const args = current.arguments?.nodes ?? []; + if (args.length !== 2) return false; + const input = staticString(program, args[0]!, seen); + // A reference relative to the base: an absolute one would not move + // with the module. + if (input === null || input.startsWith("/") || /^[a-z][a-z0-9+.-]*:/i.test(input)) return false; + return fromModuleUrl(program, args[1]!, seen); + } + return false; +} + /** Resolves the supported fork modulePath expression to an absolute source * path. Plain runtime strings are intentionally excluded: worker selection * is part of the compiled graph, not a runtime filesystem lookup. */ diff --git a/packages/compiler/src/frontend/island-worker-entry.ts b/packages/compiler/src/frontend/island-worker-entry.ts new file mode 100644 index 00000000000..bd2cdc049e3 --- /dev/null +++ b/packages/compiler/src/frontend/island-worker-entry.ts @@ -0,0 +1,135 @@ +/* Worker entries of a module whose shipped copy Node runs (scope-B: an + * island module, or a module such an island loads). Node starts a worker + * thread with its own module instances: the thread re-evaluates the entry + * and everything the entry imports. So the module tree ships the entry and + * its static import subtree as real sources (node-islands/modules.ts), and + * the Worker keeps its path: the URL the module builds from its own URL at + * build time names the same file in the tree. + * + * Only a form the tree can keep is accepted: an entry relative to the + * module's own URL (`new URL("./entry.ts", import.meta.url)`, through + * consts, templates and fileURLToPath; `import.meta.url` itself; + * `__filename`). Every other form a build can see is refused, because the + * tree keeps relative layout and nothing else: + * - a computed or unbounded entry: the build cannot know what to ship; + * - an absolute path or a `file:` URL that is not built from the module's + * own URL: the tree never holds a build-machine path; + * - a path relative to the process's working directory (Node resolves a + * plain relative filename against `cwd`): the tree is not the cwd. + * A static entry that names no file at all (a data: URL, a package + * specifier, a missing path) is Node's own business and stays as it is. + * + * A TypeScript file that ships this way must pass stripProblem: Node + * erases types in place in a worker thread too, and it refuses to do so + * inside node_modules. + * + * The scan reads only the program's own sources, and never the TypeScript + * parser: a worker entry is a program root and its import graph is in the + * program (worker-target.ts), so the files this ships are exactly the ones + * the checker bound. A module of the tree the program does not hold has no + * checked source, and its Worker stays Node's own business, as it was before + * the tree carried it. The compiler itself is compiled natively, where the + * parser island's rpc-api/rpc-process modules cannot lower — importing them + * from a lowering-side module breaks the self-hosted build. */ +import { resolve } from "node:path"; +import { trackedFileExists } from "./input-tracker.js"; +import * as ts from "./ts7/adapter.js"; +import { tsgoPath } from "./dts-paths.js"; +import { fromModuleUrl, staticForkString } from "./fork-target.js"; +import { moduleSourceCandidates } from "./module-source-candidates.js"; +import { isWorkerConstructor, workerModulePath } from "./worker-target.js"; +import type { NodeIslandWorkerScan } from "../node-islands/modules.js"; +import { stripProblem } from "../node-islands/type-strip.js"; + +export interface IslandWorkerScan { + /** The worker entries `file`'s shipped copy starts, or the reason one + * cannot ship (node-islands/modules.ts asks for each traced module). + * `fresh`: the file belongs to a worker entry's subtree, so its shipped + * copy runs in the worker thread even when the program compiles it. */ + workers(file: string, fresh: boolean): NodeIslandWorkerScan; + /** Why Node cannot run `file`'s source as shipped, or null. */ + sourceProblem(file: string): string | null; +} + +const TYPESCRIPT = /\.[mc]?ts$/; + +const NONE: NodeIslandWorkerScan = { kind: "entries", files: [] }; + +/** The scan over one program: `compiled` answers whether the program lowers + * a file natively, whose Worker constructions the native lowering answers + * (lowerWorkerNew), so outside a worker entry's subtree they never run from + * the shipped tree. Inside one (`fresh`), the worker thread evaluates the + * file's shipped copy, natively compiled or not, and its Worker + * constructions are scanned. `compiled` is asked on every call, not read + * once: the module order grows while lowering (worker targets join it) and + * the tree is collected after. */ +export function createIslandWorkerScan( + program: ts.Program, + compiled: (file: string) => boolean, +): IslandWorkerScan { + let programSources: Map | null = null; + const key = (file: string): string => tsgoPath(resolve(file)); + const sourceOf = (file: string): ts.SourceFile | null => { + programSources ??= new Map(); + if (programSources.size === 0) + for (const sf of program.getImplementationSourceFiles()) + programSources.set(key(sf.fileName), sf); + return programSources.get(key(file)) ?? null; + }; + + const workers = (file: string, fresh: boolean): NodeIslandWorkerScan => { + if (!fresh && compiled(key(file))) return NONE; + // A file outside the program has no checked source: its Worker cannot be + // recognised, and Node reports its entry as a missing module in the + // worker thread exactly as it did before the tree carried the file. + const sf = sourceOf(file); + if (sf === null) return NONE; + const entries = new Set(); + for (const construction of moduleSourceCandidates(program, sf).constructions) { + const args = construction.arguments?.nodes ?? []; + // An arity the constructor rejects is the runtime's error, in the + // island exactly as under Node. + if (args.length === 0 || args.length > 2 || args.some(ts.isSpreadElement)) continue; + if (!isWorkerConstructor(program, construction.expression)) continue; + const entry = args[0]!; + const path = workerModulePath(program, entry); + if (path === null) { + // A static entry that names no file of the build (a data: URL a + // package specifier, a filename relative to the working directory) + // is Node's to resolve; only an entry the build cannot even see is + // refused. + if (staticForkString(program, entry) !== null) continue; + return { kind: "refused", reason: refusal(sf, construction, "cannot resolve") }; + } + if (!fromModuleUrl(program, entry)) + return { + kind: "refused", + reason: refusal(sf, construction, `names the build-machine path '${path}'`), + }; + // A file that does not exist is loaded (and fails) exactly as under + // Node: there is nothing for the tree to carry. + if (trackedFileExists(path)) entries.add(path); + } + return entries.size === 0 ? NONE : { kind: "entries", files: [...entries].sort() }; + }; + + const sourceProblem = (file: string): string | null => { + if (!TYPESCRIPT.test(file)) return null; + if (/[\\/]node_modules[\\/]/.test(file)) + return `'${file}' runs in a worker thread of a Node island, but Node refuses to strip types inside node_modules`; + const sf = sourceOf(file); + if (sf === null) return null; + const problem = stripProblem(program, sf); + if (problem === null) return null; + return `'${file}' runs in a worker thread of a Node island, but Node cannot strip its types: ${problem.construct}`; + }; + + return { workers, sourceProblem }; +} + +/** A one-line description of the construction, anchored at its source: the + * collector's diagnostics carry the reason as a message. */ +function refusal(sf: ts.SourceFile, node: ts.Node, what: string): string { + const line = sf.text.slice(0, ts.tokenStartOfNode(node)).split("\n").length; + return `${sf.fileName}:${line} starts a Worker whose entry the build ${what}; a worker entry that runs in a Node island is new URL("./entry.ts", import.meta.url), so the build ships the entry and the modules it imports beside the module`; +} diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index e5299efb272..69a650bf374 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -661,6 +661,37 @@ export function collectProgram(lowerer: Lowerer, parts: FileParts[]): void { markRequireEarlyBindings(lowerer); } +/** collectIslandImport inside the collection pass. A module the island's + * shipped tree cannot carry (a computed load, an unshippable resolution) + * has its SC8006 pushed and then throws PoisonError, which a statement + * fence around a body would catch. Collection runs outside those fences, so + * the throw is caught here: the diagnostic the loader pushed is the build's + * answer, and the import stays an island load of the module. */ +function collectIslandImportAtCollection( + lowerer: Lowerer, + sf: ts.SourceFile, + stmt: ts.Statement, + spec: string, +): boolean { + try { + return collectIslandImport(lowerer, sf, stmt, spec); + } catch (error) { + if (!(error instanceof PoisonError)) throw error; + return true; + } +} + +/** collectIslandRequires inside the collection pass, for the same reason as + * collectIslandImportAtCollection: a module the shipped tree cannot carry + * reports its SC8006 and throws, and the diagnostic is the build's answer. */ +function collectIslandRequiresAtCollection(lowerer: Lowerer, sf: ts.SourceFile): void { + try { + collectIslandRequires(lowerer, sf); + } catch (error) { + if (!(error instanceof PoisonError)) throw error; + } +} + export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void { const entryPackageFile = entryPackageFilePredicate(lowerer.entry.fileName); for (const fp of parts) { @@ -724,14 +755,19 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void { // A relative import of a program module the pre-scan put in a Node // island (DESIGN.md note 1 §3.2): the island loads at this module's // position and its bindings are island values. - if (lowerer.nodeIslands !== null && collectIslandImport(lowerer, fp.sf, stmt, spec)) continue; + if ( + lowerer.nodeIslands !== null && + collectIslandImportAtCollection(lowerer, fp.sf, stmt, spec) + ) + continue; const projectDep = isRelativeSpecifier(spec) ? null : resolveImport(lowerer.program, fp.sf, spec); if (projectDep !== null) { // A Node island package that tsconfig `paths` maps to its installed // directory resolves to its declarations: it loads in Node. - if (projectDep.isDeclarationFile) collectIslandImport(lowerer, fp.sf, stmt, spec); + if (projectDep.isDeclarationFile) + collectIslandImportAtCollection(lowerer, fp.sf, stmt, spec); continue; } const npm = resolveNpmImport(fp.sf.fileName, spec); @@ -785,7 +821,7 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void { if (npm !== null && probeNodeImportRefusal(fp.sf.fileName, spec) !== null) continue; // A Node island package (scope-B NI-4): loaded in Node at this // module's position, its bindings are island values. - if (collectIslandImport(lowerer, fp.sf, stmt, spec)) continue; + if (collectIslandImportAtCollection(lowerer, fp.sf, stmt, spec)) continue; { lowerer.pushDiag(unsupportedPackageImportDiag(npm?.packageName ?? relPkg!, locOf(stmt))); continue; @@ -794,7 +830,7 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void { } for (const fp of parts) { collectDynamicImports(lowerer, fp.sf); - collectIslandRequires(lowerer, fp.sf); + collectIslandRequiresAtCollection(lowerer, fp.sf); } } diff --git a/packages/compiler/src/frontend/lowering/lower-node-island.ts b/packages/compiler/src/frontend/lowering/lower-node-island.ts index 218f7d47401..a5d12d1ea0e 100644 --- a/packages/compiler/src/frontend/lowering/lower-node-island.ts +++ b/packages/compiler/src/frontend/lowering/lower-node-island.ts @@ -278,10 +278,12 @@ function boundaryFail(lowerer: Lowerer, what: string, node: ts.Node): never { throw new PoisonError(); } -/** Whether the JavaScript file is an ES module by Node's rules. */ +/** Whether the file is an ES module by Node's rules: `.mjs` and `.mts` + * always are, `.cjs` and `.cts` never are, and any other file follows the + * nearest package.json's `type`. */ export function isEsmFile(file: string): boolean { - if (file.endsWith(".mjs")) return true; - if (file.endsWith(".cjs")) return false; + if (file.endsWith(".mjs") || file.endsWith(".mts")) return true; + if (file.endsWith(".cjs") || file.endsWith(".cts")) return false; for (let dir = dirname(file); ;) { try { const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")) as { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 10276b2186d..fa0af326061 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -820,7 +820,11 @@ import { maybePromiseUnionOf, promiseArmedUnionOf, } from "./maybe-promise.js"; -import { NodeIslandModuleCollector } from "../../node-islands/modules.js"; +import { + NodeIslandModuleCollector, + type NodeIslandWorkerScan, +} from "../../node-islands/modules.js"; +import { createIslandWorkerScan, type IslandWorkerScan } from "../island-worker-entry.js"; /** Entry function name. '%' cannot appear in a TS identifier, so a user * function can never collide with it (mangling is injective per prefix). */ @@ -4312,8 +4316,13 @@ export class Lowerer { /** Island module loads, by the importing file and specifier: the module * global holding what the import form sees (lower-node-island.ts). */ readonly islandModules = new Map(); - /** The modules Node island calls load (lower-node-island.ts). */ - readonly nodeIslandModules = new NodeIslandModuleCollector(); + /** The modules Node island calls load (lower-node-island.ts). The hooks + * answer what a module's shipped copy runs that is not an import: the + * worker entries it starts (frontend/island-worker-entry.ts) and whether + * Node can run its source (type stripping). */ + readonly nodeIslandModules: NodeIslandModuleCollector; + private islandWorkerScans: IslandWorkerScan | null = null; + private islandWorkerCompiled: { size: number; set: ReadonlySet } | null = null; /** Each call's caller literal, set to its tree path when lowering ends. */ readonly nodeIslandCallers: { literal: IrExpr & { kind: "strLit" }; caller: string }[] = []; /** Absolute specifiers, rewritten to the shipped file relative to their caller. */ @@ -4324,6 +4333,32 @@ export class Lowerer { }[] = []; /** Each island module load's path literal, set to its tree path when lowering ends. */ readonly nodeIslandLoads: { literal: IrExpr & { kind: "strLit" }; path: string }[] = []; + /** The worker entries a module its shipped copy runs starts, or why one + * cannot ship; null when the program has no Node islands. */ + private islandWorkerEntries(file: string, fresh: boolean): NodeIslandWorkerScan { + return this.islandWorkerScan().workers(file, fresh); + } + /** Why Node cannot run a module's source as shipped (type stripping). */ + private islandWorkerSourceProblem(file: string): string | null { + return this.islandWorkerScan().sourceProblem(file); + } + private islandWorkerScan(): IslandWorkerScan { + this.islandWorkerScans ??= createIslandWorkerScan(this.program, (file) => + this.compiledModulePaths().has(file), + ); + return this.islandWorkerScans; + } + /** The modules the program lowers natively, by tsgo path: rebuilt when + * the order grows (worker targets join it while lowering). */ + private compiledModulePaths(): ReadonlySet { + if (this.islandWorkerCompiled?.size !== this.moduleOrder.length) { + this.islandWorkerCompiled = { + size: this.moduleOrder.length, + set: new Set(this.moduleOrder.map((sf) => tsgoPath(resolve(sf.fileName)))), + }; + } + return this.islandWorkerCompiled.set; + } /** Island handle bindings by symbol (lower-node-island.ts). */ readonly islandHandleBindings = new Map(); /** Names a native module re-exports from an island package, by the @@ -5224,6 +5259,12 @@ export class Lowerer { "patchesStdio", "stdioPatchSites", "requestInitPropMutatedSyms", + // The Node island worker scan and the compiled-path set it is asked + // about are made on first use from the program's own modules; the set is + // made again whenever moduleOrder grows, which is a checked table, so a + // unit that grows the order has no record and both are recomputed. + "islandWorkerScans", + "islandWorkerCompiled", // familyFieldLayoutFixed and familyAnswersFromInstance answer from the // class declarations and the module order, which do not change while a // program lowers: the same questions ClassDynamicDispatch asks once when @@ -5992,6 +6033,10 @@ export class Lowerer { this.dynNodeStreams = this.generalNodeStreams && (mode.dynNodeStreams ?? false); this.dynAsyncProtocol = mode.dynAsyncProtocol ?? false; this.workerTargets = mode.workerTargets ?? []; + this.nodeIslandModules = new NodeIslandModuleCollector({ + workers: (file, fresh) => this.islandWorkerEntries(file, fresh), + sourceProblem: (file) => this.islandWorkerSourceProblem(file), + }); this.workerTargets.forEach((sf, id) => { this.workerTargetIdByPath.set(tsgoPath(resolve(sf.fileName)), id); }); diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 970398003e5..6772c306da7 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -136,7 +136,11 @@ export interface Frontend { * modules with their pre-scan markers, package import edges, and how * each refused package was refused. Call before dispose. */ nodeIslandScan?: - | ((opts: { userJs: boolean; marked?: (file: string) => boolean }) => NodeIslandScan) + | ((opts: { + userJs: boolean; + stripFor?: readonly string[] | undefined; + marked?: (file: string) => boolean; + }) => NodeIslandScan) | undefined; /** Diagnostics that do not fail a build: unknown settings in the * project's scriptc configuration (SC8004), which builds do not use. */ @@ -1394,7 +1398,7 @@ function loadFrontend( npmImportSites: npmSites, nodeIslandScan: !scanIslands ? undefined - : ({ userJs, marked }) => { + : ({ userJs, stripFor, marked }) => { const files = new Set([finalLoad.entry, ...finalLoad.moduleOrder]); addDynamicImportGraph(finalLoad, files); // Package edges matter only when the program has npm packages: one @@ -1407,6 +1411,9 @@ function loadFrontend( userJs, dropped: false, edges: npm, + ...(stripFor !== undefined && stripFor.length > 0 + ? { stripFor: new Set(stripFor) } + : {}), ...(marked === undefined ? {} : { marked }), }); return { diff --git a/packages/compiler/src/node-islands/classify.test.ts b/packages/compiler/src/node-islands/classify.test.ts new file mode 100644 index 00000000000..b89129321db --- /dev/null +++ b/packages/compiler/src/node-islands/classify.test.ts @@ -0,0 +1,82 @@ +/* Reason ids for refusal messages that carry none (DESIGN.md §1.3): the + * shapes a program meets in the wild decide whether a module may run in a + * Node island, so the table keeps them mapped to their rule. */ +import { describe, expect, test } from "vitest"; +import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import { classifyDiagnostic } from "./classify.js"; + +type Code = ScrDiagnostic["code"]; + +function refusal(code: Code, message: string): ScrDiagnostic { + return { code, message, loc: { file: "/app/src/module.ts", start: 0, end: 0 } }; +} + +const CASES: { reason: string; rule: string; code: Code; message: string }[] = [ + { + reason: "cjs-namespace", + rule: "R1.2", + code: "SC1090", + message: + "dynamic import of the program's own CommonJS module 'highlight.js/lib/index.js' (its namespace comes from module.exports through Node's CJS lexer, which has no compiled story — require it, or import it statically) is not supported yet", + }, + { + reason: "dynamic-import", + rule: "R1.2", + code: "SC1090", + message: + "dynamic import() of computed specifiers (the module graph resolves at build time — the specifier must be a string literal) are not supported yet", + }, + { + reason: "dynamic-resolve", + rule: "R1.2", + code: "SC2020", + message: + "'import.meta.resolve with a runtime-computed specifier' is part of the standard library types but has no scriptc lowering yet", + }, + { + reason: "worker-entry", + rule: "R1.2", + code: "SC2020", + message: + "'this Worker entry point' is part of the standard library types but has no scriptc lowering yet", + }, + { + reason: "host-class-base", + rule: "R1.6", + code: "SC1090", + message: "extending classes not declared in the program ('WebSocket') is not supported yet", + }, + { + reason: "dynamic-require", + rule: "R1.2", + code: "SC2020", + message: "'require' is typed by @types/node but has no scriptc lowering yet", + }, + { + reason: "dynamic-require", + rule: "R1.2", + code: "SC2020", + message: + "'createRequire's require with a computed specifier or this argument shape' is part of the standard library types but has no scriptc lowering yet", + }, +]; + +describe("refusal messages", () => { + for (const c of CASES) { + test(c.reason, () => { + const reason = classifyDiagnostic(refusal(c.code, c.message), "refusal"); + expect(reason.reason).toBe(c.reason); + expect(reason.rule).toBe(c.rule); + expect(reason.class).toBe("runtime"); + }); + } + + test("a construct the runtime cannot help with stays a gap", () => { + const reason = classifyDiagnostic( + refusal("SC1090", "union discriminant reads are not supported yet"), + "refusal", + ); + expect(reason.class).toBe("gap"); + expect(reason.rule).toBeNull(); + }); +}); diff --git a/packages/compiler/src/node-islands/classify.ts b/packages/compiler/src/node-islands/classify.ts index b08ac63aaee..00a7db1759a 100644 --- a/packages/compiler/src/node-islands/classify.ts +++ b/packages/compiler/src/node-islands/classify.ts @@ -11,18 +11,44 @@ import { classOfRule, ruleInfo, ruleOfReason } from "./rules.js"; import type { RefusalRecord } from "./scan.js"; import type { NodeIslandReason, NodeIslandReasonSource } from "./summary.js"; -/** Message shapes of refusal sites that predate reason ids. */ -const MESSAGE_REASONS: { code: string; pattern: RegExp; reason: string }[] = [ +/** Message shapes of refusal sites that predate reason ids. A shape that + * one refusal site reports as SC1090 and another as SC2020 names both. */ +const MESSAGE_REASONS: { code: string | readonly string[]; pattern: RegExp; reason: string }[] = [ { code: "SC2010", pattern: /./, reason: "code-from-string" }, { code: "SC1010", pattern: /'(?:node:)?vm'/, reason: "vm" }, { code: "SC1090", pattern: /^dynamic import\(\) of computed/, reason: "dynamic-import" }, + { + code: "SC1090", + pattern: /^dynamic import of the program's own CommonJS module /, + reason: "cjs-namespace", + }, { code: "SC1090", pattern: /require\(\) with a computed specifier/, reason: "dynamic-require" }, + { + code: ["SC1090", "SC2020"], + pattern: /^'import\.meta\.resolve with a runtime-computed specifier'/, + reason: "dynamic-resolve", + }, + { + code: ["SC1090", "SC2020"], + pattern: /^'?this Worker entry point/, + reason: "worker-entry", + }, + { + code: "SC1090", + pattern: /^extending classes not declared in the program/, + reason: "host-class-base", + }, { code: "SC1090", pattern: /'with' statement/, reason: "with" }, { code: "SC1090", pattern: /sloppy-mode parameter aliases/, reason: "sloppy-arguments" }, { code: "SC2020", pattern: /^'(?:eval|Function)'/, reason: "code-from-string" }, { code: "SC2020", - pattern: /createRequire's require with this argument shape/, + pattern: /createRequire's require with (?:a computed specifier or )?this argument shape/, + reason: "dynamic-require", + }, + { + code: "SC2020", + pattern: /^'require' is typed by @types\/node/, reason: "dynamic-require", }, { code: "SC2020", pattern: /^native addon /, reason: "native-addon" }, @@ -43,7 +69,8 @@ const MESSAGE_REASONS: { code: string; pattern: RegExp; reason: string }[] = [ function reasonIdOf(d: ScrDiagnostic): string | null { if (d.reason !== undefined) return d.reason; for (const entry of MESSAGE_REASONS) { - if (entry.code === d.code && entry.pattern.test(d.message)) return entry.reason; + const codes = typeof entry.code === "string" ? [entry.code] : entry.code; + if (codes.includes(d.code) && entry.pattern.test(d.message)) return entry.reason; } return null; } diff --git a/packages/compiler/src/node-islands/detect.test.ts b/packages/compiler/src/node-islands/detect.test.ts index 06d34396e81..e3a91e2d4c5 100644 --- a/packages/compiler/src/node-islands/detect.test.ts +++ b/packages/compiler/src/node-islands/detect.test.ts @@ -399,6 +399,8 @@ test("CommonJS object reads are recorded for the patch check (H3.4)", () => { expect(reads("const fs = require('fs'); fs['readFileSync'](p);")).toEqual(["fs.readFileSync"]); expect(reads("const fs = require('fs'); fs[name](p);")).toEqual(["fs.*"]); expect(reads("const fs = require('fs'); fs['readFileSync'] = f;")).toEqual([]); + // A require() read directly is a read through the object too. + expect(reads("require('fs').readFileSync(p);")).toEqual(["fs.readFileSync"]); // Destructuring reads each property it takes when it runs. expect(reads("const { readFileSync } = require('node:fs');")).toEqual(["fs.readFileSync"]); expect(reads("const fs = require('fs'); const { readFileSync: read, ...rest } = fs;")).toEqual([ @@ -409,11 +411,38 @@ test("CommonJS object reads are recorded for the patch check (H3.4)", () => { "fs.promises", "fs.promises.readFile", ]); + // A destructuring may start at a member object of the module, spelled as a + // subpath spec or through the CommonJS object. + expect(reads("const { readFile } = require('fs/promises');")).toEqual(["fs.promises.readFile"]); + expect(reads("const { readFile } = require('node:fs/promises');")).toEqual([ + "fs.promises.readFile", + ]); + expect(reads("const { readFile } = require('fs').promises;")).toEqual([ + "fs.promises.readFile", + "fs.promises", + ]); + expect(reads("const fs = require('fs'); const { readFile } = fs.promises;")).toEqual([ + "fs.promises.readFile", + "fs.promises", + ]); + expect(reads("const fs = require('fs'); const { readFile, ...rest } = fs.promises;")).toEqual([ + "fs.promises.readFile", + "fs.*", + "fs.promises", + ]); + // A require() the scan cannot resolve to a module is not one, and a + // subpath module that is not a member object reads nothing of the module + // its spec starts with. + expect(reads("const { readFile } = require(name);")).toEqual([]); + expect(reads("const { ReadableStream } = require('stream/web');")).toEqual([]); expect( reads("import fs from 'node:fs'; let readFileSync; ({ readFileSync } = fs);", "mjs"), ).toEqual(["fs.readFileSync"]); // A namespace import is not the CommonJS object. expect(reads("import * as fs from 'node:fs'; const { readFileSync } = fs;", "mjs")).toEqual([]); + expect(reads("import * as fs from 'node:fs'; const { readFile } = fs.promises;", "mjs")).toEqual( + [], + ); }); test("global reads are recorded for the implicit-global check (H3.4)", () => { diff --git a/packages/compiler/src/node-islands/detect.ts b/packages/compiler/src/node-islands/detect.ts index c23fd99b501..876c3f1849f 100644 --- a/packages/compiler/src/node-islands/detect.ts +++ b/packages/compiler/src/node-islands/detect.ts @@ -163,6 +163,22 @@ function builtinModuleName(spec: string): string | null { return null; } +/** Subpath modules Node serves as objects of their own rather than as a + * member of the module the spec starts with. */ +const STANDALONE_SUBPATH_MODULES = new Set(["stream/web", "stream/consumers"]); + +/** The member object a builtin's subpath spec names: `require("fs/promises")` + * is `require("fs").promises`, and every other subpath of a supported + * builtin is a member object too. Null for the module itself and for a + * standalone subpath module, whose members belong to no object of the + * module the spec starts with. */ +function memberBaseOf(module: string): string | null { + const slash = module.indexOf("/"); + if (slash < 0) return module; + if (STANDALONE_SUBPATH_MODULES.has(module)) return null; + return `${module.slice(0, slash)}.${module.slice(slash + 1)}`; +} + /** Parentheses and the TypeScript wrappers Node erases (`as`, * `satisfies`, `!` and the angle-bracket assertion). */ function stripWrappers(node: ts.Node): ts.Node { @@ -685,6 +701,23 @@ class ModuleScanner { ); } + /** A member object of a builtin module's CommonJS object + * (`require("fs").promises`, `fs.promises`): its own properties are read + * through the same object, so `const { readFile } = fs.promises` reads + * `fs.promises.readFile`. A namespace import is still not that object. */ + private isCjsObjectMember(value: ts.Node): boolean { + const node = stripWrappers(value); + if (!ts.isPropertyAccessExpression(node)) return false; + const root = chainRoot(node); + if (root !== null) return this.isCjsObjectValue(root); + // A chain that starts at the require() call itself + // (`require("fs").promises`). + let base = stripWrappers(node.expression); + while (ts.isPropertyAccessExpression(base) || ts.isElementAccessExpression(base)) + base = stripWrappers(base.expression); + return this.isCjsObjectValue(base); + } + /** `require("fs")`, `require("node:util").inherits` → "fs", "util.inherits". */ private builtinOf(node: ts.Node): string | null { const value = stripWrappers(node); @@ -985,11 +1018,24 @@ class ModuleScanner { * so it is recorded as the whole module ("fs.*"). */ private recordCjsRead(node: ts.PropertyAccessExpression | ts.ElementAccessExpression): void { const root = chainRoot(node); - if (root === null) return; - const binding = this.bindingAt(root.text, root); - if (binding === undefined || !binding.cjs || binding.builtin === null) return; - const module = binding.builtin; const text = chainText(node); + let member: string; + if (root === null) { + // A require() of the builtin read directly + // (`require("fs").readFileSync`): the chain starts at the call. + let base = stripWrappers(node.expression); + while (ts.isPropertyAccessExpression(base) || ts.isElementAccessExpression(base)) + base = stripWrappers(base.expression); + if (!this.isCjsObjectValue(base)) return; + const module = this.builtinOf(base); + if (module === null) return; + member = this.builtinOf(node) ?? `${module.split(".")[0]}.*`; + } else { + const binding = this.bindingAt(root.text, root); + if (binding === undefined || !binding.cjs || binding.builtin === null) return; + const module = binding.builtin; + member = text === null ? `${module}.*` : module + text.slice(root.text.length); + } const parent = node.parent; if (parent !== undefined) { if ( @@ -1005,7 +1051,7 @@ class ModuleScanner { ) return; } - this.pushCjsRead(text === null ? `${module}.*` : module + text.slice(root.text.length), node); + this.pushCjsRead(member, node); } private pushCjsRead(member: string, node: ts.Node): void { @@ -1019,14 +1065,22 @@ class ModuleScanner { * CommonJS object (`const { readFileSync } = require("fs")`, * `({ readFileSync } = fs)`): each property it takes is read when the * destructuring runs, so it sees a patch installed before. A rest element - * or a computed key reads members the scan cannot name ("fs.*"). */ + * or a computed key reads members the scan cannot name ("fs.*"). The + * source may be one of the module's member objects + * (`const { readFile } = require("fs/promises")`, `= fs.promises`): the + * pattern then names the member below it. */ private recordCjsPatternReads(pattern: ts.Node, source: ts.Node | undefined): void { if (source === undefined) return; const value = stripWrappers(source); - if (!this.isCjsObjectValue(value)) return; + if (!this.isCjsObjectValue(value) && !this.isCjsObjectMember(value)) return; const module = this.builtinOf(value); - if (module === null || module.includes(".")) return; - this.recordPatternReads(pattern, module); + if (module === null) return; + // A subpath spec names the member object Node hangs off the module + // (`require("fs/promises")` is `require("fs").promises`), so the reads + // are spelled through the module the object belongs to. + const base = memberBaseOf(module); + if (base === null) return; + this.recordPatternReads(pattern, base); } private recordPatternReads(pattern: ts.Node, base: string): void { diff --git a/packages/compiler/src/node-islands/effect-gate.ts b/packages/compiler/src/node-islands/effect-gate.ts index d99d831cc34..fa6d8527e1f 100644 --- a/packages/compiler/src/node-islands/effect-gate.ts +++ b/packages/compiler/src/node-islands/effect-gate.ts @@ -181,8 +181,8 @@ export type EffectGateResult = | { ok: false; reason: string; file: string }; function isEsm(file: string): boolean { - if (file.endsWith(".mjs")) return true; - if (file.endsWith(".cjs")) return false; + if (file.endsWith(".mjs") || file.endsWith(".mts")) return true; + if (file.endsWith(".cjs") || file.endsWith(".cts")) return false; for (let dir = dirname(file); ;) { try { const pkg = JSON.parse(readFileSync(resolve(dir, "package.json"), "utf8")) as { diff --git a/packages/compiler/src/node-islands/modules.test.ts b/packages/compiler/src/node-islands/modules.test.ts index 35f09d4c5ec..f49b08dc701 100644 --- a/packages/compiler/src/node-islands/modules.test.ts +++ b/packages/compiler/src/node-islands/modules.test.ts @@ -548,6 +548,76 @@ describe("modules the program provides", () => { expect(files.some((f) => f.path.endsWith("internal.mjs"))).toBe(false); }); + test("a worker entry a module starts ships as source, and so does what it imports", () => { + const root = project({ + "src/main.ts": "", + "src/launcher.mjs": [ + 'import { Worker } from "node:worker_threads";', + 'export const start = () => new Worker(new URL("./worker.mjs", import.meta.url));', + ].join("\n"), + "src/worker.mjs": 'import { count } from "./state.mjs";\ncount();', + "src/state.mjs": "export let count = 0;\nexport const internal = 7;\n", + }); + const collector = new NodeIslandModuleCollector({ + workers: (file) => + file.endsWith("launcher.mjs") + ? { kind: "entries", files: [join(root, "src/worker.mjs")] } + : { kind: "entries", files: [] }, + }); + // A worker thread has its own module instances, so the program module + // the entry imports ships as source there even though the island thread + // reads a placeholder for it. + collector.addProvided(join(root, "src/state.mjs")); + expect(collector.add(join(root, "src/main.ts"), "./launcher.mjs")).toBeNull(); + const tree = collector.tree()!; + expect(tree.root).toBe(join(root, "src")); + expect(tree.files.map((f) => f.path)).toEqual(["launcher.mjs", "state.mjs", "worker.mjs"]); + expect(tree.files.every((f) => f.placeholder !== true)).toBe(true); + const data = readNodeIslandModules(tree); + if (typeof data === "string") throw new Error(data); + expect(new TextDecoder().decode(data)).toContain("export const internal = 7;"); + }); + + test("a worker entry traced after the placeholder ships the real source instead", () => { + const root = project({ + "src/main.ts": "", + "src/plugin.mjs": [ + 'import { s } from "./shared.mjs";', + 'new Worker(new URL("./worker.mjs", import.meta.url));', + ].join("\n"), + "src/worker.mjs": 'import { s } from "./shared.mjs";', + "src/shared.mjs": "export const s = 42;\n", + }); + const collector = new NodeIslandModuleCollector({ + workers: (file) => + file.endsWith("plugin.mjs") + ? { kind: "entries", files: [join(root, "src/worker.mjs")] } + : { kind: "entries", files: [] }, + }); + collector.addProvided(join(root, "src/shared.mjs")); + expect(collector.add(join(root, "src/main.ts"), "./plugin.mjs")).toBeNull(); + const shared = collector.tree()!.files.find((f) => f.path === "shared.mjs")!; + expect(shared.placeholder).toBeUndefined(); + }); + + test("a worker entry the build refuses stops the collection", () => { + const root = project({ + "src/main.ts": "", + "src/plugin.mjs": 'new Worker(new URL("./worker.ts", import.meta.url));', + "src/worker.ts": "enum E {}\n", + }); + const collector = new NodeIslandModuleCollector({ + workers: (file) => + file.endsWith("plugin.mjs") + ? { kind: "entries", files: [join(root, "src/worker.ts")] } + : { kind: "entries", files: [] }, + sourceProblem: (file) => `${file}: Node cannot strip its types: an enum`, + }); + expect(collector.add(join(root, "src/main.ts"), "./plugin.mjs")).toBe( + `${join(root, "src/worker.ts")}: Node cannot strip its types: an enum`, + ); + }); + test("requireModule specifiers come from code, not comments or strings", () => { expect( requireModuleSpecifiers(` diff --git a/packages/compiler/src/node-islands/modules.ts b/packages/compiler/src/node-islands/modules.ts index 82cd3f9e9b2..72cb42a7990 100644 --- a/packages/compiler/src/node-islands/modules.ts +++ b/packages/compiler/src/node-islands/modules.ts @@ -163,6 +163,26 @@ function portable(path: string): string { return sep === "/" ? path : path.split(sep).join("/"); } +/** What a module whose shipped copy Node runs starts, or why it cannot + * ship: the worker entries it resolves to a file (worker threads, Node's + * own: the worker re-evaluates its entry in its own thread, so the tree + * ships the entry and its static import subtree as real sources). */ +export type NodeIslandWorkerScan = + | { kind: "entries"; files: readonly string[] } + | { kind: "refused"; reason: string }; + +export interface NodeIslandModuleHooks { + /** The worker entries `file`'s source starts, or the reason one cannot + * ship. `fresh` is true for a file of a worker entry's subtree, whose real + * source runs in the worker's own thread even when the program compiles + * the file too; otherwise a module the program compiles lowers its Worker + * constructions natively and its shipped copy starts none. */ + workers?: ((file: string, fresh: boolean) => NodeIslandWorkerScan) | undefined; + /** Why Node cannot run `file`'s source as shipped (type stripping), or + * null. Asked for every file a worker entry's subtree ships. */ + sourceProblem?: ((file: string) => string | null) | undefined; +} + export class NodeIslandModuleCollector { /** Real paths of collected files. */ private readonly files = new Set(); @@ -176,6 +196,12 @@ export class NodeIslandModuleCollector { private readonly loaded = new Set(); /** Program modules whose source the program provides (addProvided). */ private readonly provided = new Set(); + /** Files a worker entry's static import subtree runs: shipped as their + * real sources, even where the program provides a placeholder. */ + private readonly fresh = new Set(); + /** Fresh files whose subtree is already traced (fresh reaches further + * than the island-side trace: it also runs through placeholders). */ + private readonly freshTraced = new Set(); /** The problems addNamedData reported (isDataFileProblem). */ private readonly dataProblems = new Set(); @@ -185,6 +211,8 @@ export class NodeIslandModuleCollector { return this.dataProblems.has(problem); } + constructor(private readonly hooks: NodeIslandModuleHooks = {}) {} + /** Collects the module `specifier` names from `caller` (a real path). * Returns null, or the reason the build cannot ship it. */ add(caller: string, specifier: string): string | null { @@ -241,7 +269,11 @@ export class NodeIslandModuleCollector { const files: NodeIslandModuleFile[] = []; for (const file of [...this.files].sort()) { const path = portable(relative(root, file)); - if (this.provided.has(file)) { + // A worker entry's subtree runs in the worker's own thread, with its + // own instances: its real source ships even where the program's copy + // is a placeholder (the island thread never reads the placeholder + // anyway; the loader hooks answer it). + if (this.provided.has(file) && !this.fresh.has(file)) { const bytes = providedModulePlaceholder(path); files.push({ path, size: bytes.length, sha256: sha256(bytes), placeholder: true }); continue; @@ -270,7 +302,7 @@ export class NodeIslandModuleCollector { return tree === null ? file : `./${portable(relative(tree.root, file))}`; } - private resolveFrom(fromFile: string, specifier: string): string | null { + private resolveFrom(fromFile: string, specifier: string, fresh = false): string | null { if (isBuiltin(specifier)) return null; if (specifier.startsWith("#")) return `'${specifier}' is a package import ("imports" in package.json), which Node island modules of the program cannot use`; @@ -281,7 +313,8 @@ export class NodeIslandModuleCollector { // optional require in a try block keeps working). if (isPathSpecifier(specifier)) { const target = resolveFile(resolve(dirname(fromFile), specifier)); - return target === null ? null : this.addModule(this.placed(target)); + if (target === null) return null; + return fresh ? this.addFresh(this.placed(target)) : this.addModule(this.placed(target)); } const dir = packageDirectory(dirname(fromFile), packageName(specifier)); if (dir !== null) this.addPackage(dir); @@ -327,6 +360,29 @@ export class NodeIslandModuleCollector { // The program provides its source: island code never runs this file, // so what it imports is not shipped for it. if (this.provided.has(file)) return null; + return this.traceSource(file, false); + } + + /** A worker entry and, through its imports, the subtree the worker's own + * thread evaluates: shipped as its real sources, placeholders or not, + * because the worker has its own module instances (Node starts a + * worker's module graph afresh, in a thread whose loader hooks are its + * own). */ + private addFresh(file: string): string | null { + this.files.add(file); + this.fresh.add(file); + if (this.freshTraced.has(file)) return null; + this.freshTraced.add(file); + this.traced.add(file); + const problem = this.hooks.sourceProblem?.(file) ?? null; + if (problem !== null) return problem; + return this.traceSource(file, true); + } + + /** The source of a collected program module: its package scope, the data + * files it names, its module specifiers (`fresh` on a worker entry's + * subtree) and the worker entries it starts. */ + private traceSource(file: string, fresh: boolean): string | null { const scope = packageScope(dirname(file)); if (scope !== null) this.files.add(scope); if (!TRACED.test(file)) return null; @@ -341,7 +397,14 @@ export class NodeIslandModuleCollector { return `${file}:${line} loads a module through ${first.form}; a module that runs in a Node island loads modules by string-literal specifiers only`; } for (const specifier of scan.specifiers) { - const problem = this.resolveFrom(file, specifier); + const problem = this.resolveFrom(file, specifier, fresh); + if (problem !== null) return problem; + } + const workers = this.hooks.workers?.(file, fresh); + if (workers === undefined) return null; + if (workers.kind === "refused") return workers.reason; + for (const entry of workers.files) { + const problem = this.addFresh(this.placed(entry)); if (problem !== null) return problem; } return null; diff --git a/packages/compiler/src/node-islands/rules.ts b/packages/compiler/src/node-islands/rules.ts index 52af539d86e..63060f2183d 100644 --- a/packages/compiler/src/node-islands/rules.ts +++ b/packages/compiler/src/node-islands/rules.ts @@ -58,11 +58,15 @@ const REASON_RULES: Record = { vm: "R1.1", "dynamic-require": "R1.2", "dynamic-import": "R1.2", + "dynamic-resolve": "R1.2", + "worker-entry": "R1.2", + "cjs-namespace": "R1.2", "unresolved-import": "R1.2", "native-addon": "R1.3", "loader-hook": "R1.4", "engine-internal": "R1.5", "weak-ref": "R1.6", + "host-class-base": "R1.6", "host-global": "R1.6", "constructor-function": "R2.1", "prototype-write": "R2.2", diff --git a/packages/compiler/src/node-islands/scan.ts b/packages/compiler/src/node-islands/scan.ts index a5cb2426c2a..416fcbfe02b 100644 --- a/packages/compiler/src/node-islands/scan.ts +++ b/packages/compiler/src/node-islands/scan.ts @@ -195,7 +195,9 @@ function sloppyParsable(program: ts.Program, sf: ts.SourceFile): boolean { /** Scans `files`: markers for npm JavaScript (and for user JavaScript when * `userJs`), and package edges for every file when `edges` (a program with - * no npm package has none to find). */ + * no npm package has none to find). `stripFor` names user TypeScript modules + * a build is considering for an island: their `stripProblem` is computed so + * the build can refuse a file Node cannot run as written. */ export function scanFiles( program: ts.Program, files: readonly ts.SourceFile[], @@ -203,6 +205,7 @@ export function scanFiles( userJs: boolean; dropped: boolean; edges: boolean; + stripFor?: ReadonlySet | undefined; /** Whether the policy marks a user module to run in a Node island * (`--node-island=`, `scriptc.nodeIslands.modules`). */ marked?: (file: string) => boolean; @@ -222,6 +225,10 @@ export function scanFiles( // (H3.2): its other markers are JavaScript semantics, and sloppy // parsing is never required of TypeScript. const userTs = !js && origin === "user" && !sf.isDeclarationFile && opts.userJs; + // A module a build asks about for an island (H3.5's check, before it + // decides): the cost stays with the modules under consideration. + const forRun = + !js && origin === "user" && !sf.isDeclarationFile && opts.stripFor?.has(sf.fileName) === true; // A TypeScript module the policy marks runs in Node with its types // stripped, like one that islands for a builtin patch. const markedTs = @@ -242,9 +249,10 @@ export function scanFiles( globalReads: facts.globalReads, // A TypeScript module can island for a builtin patch (H3.2) and for a // read of a member another island's patch changes (H3.4): both need - // Node to run the file with type stripping only. + // Node to run the file with type stripping only. A module a build is + // considering for an island (H3.5) needs the answer before it decides. stripProblem: - (userTs && (markers.length > 0 || facts.cjsReads.length > 0)) || markedTs + forRun || (userTs && (markers.length > 0 || facts.cjsReads.length > 0)) || markedTs ? stripProblem(program, sf) : null, dropped: opts.dropped, diff --git a/packages/compiler/src/node-islands/summary.ts b/packages/compiler/src/node-islands/summary.ts index c7339956149..035732fcab6 100644 --- a/packages/compiler/src/node-islands/summary.ts +++ b/packages/compiler/src/node-islands/summary.ts @@ -82,6 +82,8 @@ export interface NodeIslandReport { * addition to the pre-scan construct. */ const REASON_SENTENCES: Readonly> = { "builtin-patch": "it patches a builtin module's CommonJS export, which only code in Node can see", + "host-class-base": + "it extends a class only the JavaScript runtime has, and native code cannot inherit it", }; /** "R2.1 constructor functions: 'new Foo()' of a function" or diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6900-node-island-host-class/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6900-node-island-host-class/main.ts.json new file mode 100644 index 00000000000..182b1b29809 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6900-node-island-host-class/main.ts.json @@ -0,0 +1,7 @@ +{ + "order": [ + "/tests/corpus/6900-node-island-host-class/socket.ts", + "/tests/corpus/6900-node-island-host-class/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6901-island-worker-entry/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6901-island-worker-entry/main.ts.json new file mode 100644 index 00000000000..aa7d166e724 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6901-island-worker-entry/main.ts.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/6901-island-worker-entry/state.ts", + "/tests/corpus/6901-island-worker-entry/launcher.ts", + "/tests/corpus/6901-island-worker-entry/main.ts" + ], + "diags": [] +} diff --git a/tests/corpus/6900-node-island-host-class/globals.d.ts b/tests/corpus/6900-node-island-host-class/globals.d.ts new file mode 100644 index 00000000000..84fd4677e2b --- /dev/null +++ b/tests/corpus/6900-node-island-host-class/globals.d.ts @@ -0,0 +1,12 @@ +// The program declares the host class the way @types/node declares the +// WebSocket global: an interface plus a `var` of its constructor type. Only +// Node has the value itself. +interface HostSocket { + readonly readyState: number; + close(code?: number, reason?: string): void; +} + +declare var WebSocket: { + new (url: string): HostSocket; + readonly prototype: HostSocket; +}; diff --git a/tests/corpus/6900-node-island-host-class/main.ts b/tests/corpus/6900-node-island-host-class/main.ts new file mode 100644 index 00000000000..bc25e4724bf --- /dev/null +++ b/tests/corpus/6900-node-island-host-class/main.ts @@ -0,0 +1,7 @@ +// @requires-libnode +// socket.ts extends a host class the program only declares, and only Node +// has the class itself: the module runs in a Node island automatically +// (R1.6) while the entry stays native. +import { describeSocket } from "./socket.ts"; + +console.log("socket:", describeSocket()); diff --git a/tests/corpus/6900-node-island-host-class/package.json b/tests/corpus/6900-node-island-host-class/package.json new file mode 100644 index 00000000000..6d563eb9043 --- /dev/null +++ b/tests/corpus/6900-node-island-host-class/package.json @@ -0,0 +1 @@ +{ "name": "host-class-island", "private": true, "type": "module" } diff --git a/tests/corpus/6900-node-island-host-class/socket.ts b/tests/corpus/6900-node-island-host-class/socket.ts new file mode 100644 index 00000000000..b4b901c576f --- /dev/null +++ b/tests/corpus/6900-node-island-host-class/socket.ts @@ -0,0 +1,12 @@ +/// +// A class whose base only the JavaScript runtime has (the census's +// `class extends WebSocket` shape): native lowering cannot extend it, so +// this module runs in a Node island automatically (R1.6) and the entry +// stays native. Nothing connects: the program reads the class itself. +export class Socket extends WebSocket { + static tag = "socket"; +} + +export function describeSocket(): string { + return `${Socket.tag}:${Object.getPrototypeOf(Socket) === WebSocket}`; +} diff --git a/tests/corpus/6901-island-worker-entry/globals.d.ts b/tests/corpus/6901-island-worker-entry/globals.d.ts new file mode 100644 index 00000000000..84fd4677e2b --- /dev/null +++ b/tests/corpus/6901-island-worker-entry/globals.d.ts @@ -0,0 +1,12 @@ +// The program declares the host class the way @types/node declares the +// WebSocket global: an interface plus a `var` of its constructor type. Only +// Node has the value itself. +interface HostSocket { + readonly readyState: number; + close(code?: number, reason?: string): void; +} + +declare var WebSocket: { + new (url: string): HostSocket; + readonly prototype: HostSocket; +}; diff --git a/tests/corpus/6901-island-worker-entry/launcher.ts b/tests/corpus/6901-island-worker-entry/launcher.ts new file mode 100644 index 00000000000..8aac4bf74dc --- /dev/null +++ b/tests/corpus/6901-island-worker-entry/launcher.ts @@ -0,0 +1,34 @@ +/// +// Runs in a Node island: it extends a host class only Node has, so the +// native lowering refuses the module and the build runs it in Node (the +// 6900 mechanism). The Worker it starts is Node's own: the build ships +// worker.ts and what it imports beside this module, and the URL this module +// builds at run time names the same file in the shipped tree. +import { Worker } from "node:worker_threads"; + +export class Socket extends WebSocket { + static tag = "socket"; +} + +// The program reads the class, so the refusable base is live code: the +// build islanded this module rather than inventing a native base. +export function socketTag(): string { + return `${Socket.tag}:${Object.getPrototypeOf(Socket) === WebSocket}`; +} + +export async function roundTrip(value: number): Promise { + const worker = new Worker(new URL("./worker.ts", import.meta.url)); + const reply = (): Promise => + new Promise((resolve, reject) => { + worker.once("message", resolve); + worker.once("error", reject); + }); + const firstMessage = reply(); + worker.postMessage(value); + const first = await firstMessage; + const secondMessage = reply(); + worker.postMessage(value + 1); + const second = await secondMessage; + await worker.terminate(); + return `${first} ${second}`; +} diff --git a/tests/corpus/6901-island-worker-entry/main.ts b/tests/corpus/6901-island-worker-entry/main.ts new file mode 100644 index 00000000000..077216cdf5d --- /dev/null +++ b/tests/corpus/6901-island-worker-entry/main.ts @@ -0,0 +1,13 @@ +// @requires-libnode +// A Worker the program starts from a module that runs in a Node island: +// its entry is a module of the program, so the island tree ships the entry +// and what it imports as real sources, and the worker thread evaluates its +// own copy (Node gives every thread its own module instances). The program's +// copy of state.ts and the worker's copy count separately. +import { bump, count } from "./state.ts"; +import { roundTrip, socketTag } from "./launcher.ts"; + +console.log("socket:", socketTag()); +console.log("main:", bump(), bump(), count); +console.log("worker:", await roundTrip(21)); +console.log("main after:", count); diff --git a/tests/corpus/6901-island-worker-entry/package.json b/tests/corpus/6901-island-worker-entry/package.json new file mode 100644 index 00000000000..76a7f3fc582 --- /dev/null +++ b/tests/corpus/6901-island-worker-entry/package.json @@ -0,0 +1 @@ +{ "name": "island-worker-entry", "private": true, "type": "module" } diff --git a/tests/corpus/6901-island-worker-entry/state.ts b/tests/corpus/6901-island-worker-entry/state.ts new file mode 100644 index 00000000000..f11378093b2 --- /dev/null +++ b/tests/corpus/6901-island-worker-entry/state.ts @@ -0,0 +1,9 @@ +// One module, two threads: the program compiles this copy for its own +// thread, and the worker's thread evaluates the island tree's copy, because +// Node gives every thread its own module instances. +export let count = 0; + +export function bump(): number { + count++; + return count; +} diff --git a/tests/corpus/6901-island-worker-entry/worker.ts b/tests/corpus/6901-island-worker-entry/worker.ts new file mode 100644 index 00000000000..9192e7d13ef --- /dev/null +++ b/tests/corpus/6901-island-worker-entry/worker.ts @@ -0,0 +1,9 @@ +// The worker entry, shipped as real source with the island tree: this +// thread's copy of state.ts starts at zero, whatever the program's copy +// holds. +import { parentPort } from "node:worker_threads"; +import { bump, count } from "./state.ts"; + +parentPort?.on("message", (value: number) => { + parentPort?.postMessage(`echo=${value} worker=${bump()},${bump()},${bump()} count=${count}`); +}); diff --git a/tests/harness/__snapshots__/coverage-node-islands-fenced-sdk-values.txt b/tests/harness/__snapshots__/coverage-node-islands-fenced-sdk-values.txt index f7b1bcb8454..0341b90e0f7 100644 --- a/tests/harness/__snapshots__/coverage-node-islands-fenced-sdk-values.txt +++ b/tests/harness/__snapshots__/coverage-node-islands-fenced-sdk-values.txt @@ -8,8 +8,8 @@ scriptc coverage tests/coverage-fixtures/node-islands-fenced-sdk-values/main.ts ni-sdk static Node islands (would-be, report-only: the build is unchanged; --node-islands=auto) - gap ni-sdk@2.0.0 - tests/coverage-fixtures/node-islands-fenced-sdk-values/node_modules/ni-sdk/index.js:7:26: SC1090: extending classes not declared in the program ('Promise') is not supported yet + runtime ni-sdk@2.0.0 + tests/coverage-fixtures/node-islands-fenced-sdk-values/node_modules/ni-sdk/index.js:7:26: R1.6 host-class-base: it extends a class only the JavaScript runtime has, and native code cannot inherit it (… hint: scriptc coverage --explain-island= lists every reason fully static — all analyzed statements compile to native code. \ No newline at end of file diff --git a/tests/harness/corpus-costs/6900-node-island-host-class/main.ts.json b/tests/harness/corpus-costs/6900-node-island-host-class/main.ts.json new file mode 100644 index 00000000000..d7dc7bf24d8 --- /dev/null +++ b/tests/harness/corpus-costs/6900-node-island-host-class/main.ts.json @@ -0,0 +1 @@ +{ "plain": 8000, "san": 45000 } diff --git a/tests/harness/corpus-costs/6901-island-worker-entry/main.ts.json b/tests/harness/corpus-costs/6901-island-worker-entry/main.ts.json new file mode 100644 index 00000000000..0fd8341bf23 --- /dev/null +++ b/tests/harness/corpus-costs/6901-island-worker-entry/main.ts.json @@ -0,0 +1 @@ +{ "plain": 8000, "san": 50000 } diff --git a/tests/harness/coverage.test.ts b/tests/harness/coverage.test.ts index 0cebd3c819d..9b293ddf7b7 100644 --- a/tests/harness/coverage.test.ts +++ b/tests/harness/coverage.test.ts @@ -1105,8 +1105,12 @@ test("a package a build islands by its module-init fence is the same island here const { coverage } = analyze(entry); expect(coverage.diagnostics).toEqual([]); expect(coverage.stats.statementsFailed).toBe(0); + // A Promise subclass needs the JavaScript runtime, and the reason now + // carries the rule that says so (R1.6 host-class-base), so the island is + // classed "runtime" rather than the unnamed "gap" a refusal without a + // reason id falls back to. expect(coverage.nodeIslands?.islands.map((i) => [i.package, i.class])).toEqual([ - ["ni-sdk", "gap"], + ["ni-sdk", "runtime"], ]); // The build decides the same island: its SC8001 note names the package the // report lists. diff --git a/tests/harness/node-island.test.ts b/tests/harness/node-island.test.ts index 5c674affa9e..2aceb0297ee 100644 --- a/tests/harness/node-island.test.ts +++ b/tests/harness/node-island.test.ts @@ -93,6 +93,26 @@ async function build(name: string, libnode?: string, entry = fixture): Promise(build: () => Promise): Promise { + const cache = mkdtempSync(join(tmpdir(), "scriptc-builtin-patch-cache-")); + const previous = process.env["SCRIPTC_CACHE_DIR"]; + process.env["SCRIPTC_CACHE_DIR"] = cache; + try { + return await build(); + } finally { + if (previous === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = previous; + rmSync(cache, { recursive: true, force: true }); + } +} + +/** A build's diagnostics as "SC0000: message" lines. */ +function messages(result: { diagnostics?: { code: string; message: string }[] }): string[] { + return (result.diagnostics ?? []).map((d) => `${d.code}: ${d.message}`); +} + /** A bridge library that answers the ABI handshake and nothing else. */ async function fakePack(name: string, abi: string, version: string): Promise { const dir = join(outDir, name); @@ -371,23 +391,6 @@ describe.skipIf(skip)("Node islands in-process", () => { ); } - async function withFreshBuildCache(build: () => Promise): Promise { - const cache = mkdtempSync(join(tmpdir(), "scriptc-builtin-patch-cache-")); - const previous = process.env["SCRIPTC_CACHE_DIR"]; - process.env["SCRIPTC_CACHE_DIR"] = cache; - try { - return await build(); - } finally { - if (previous === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; - else process.env["SCRIPTC_CACHE_DIR"] = previous; - rmSync(cache, { recursive: true, force: true }); - } - } - - function messages(result: { diagnostics?: { code: string; message: string }[] }): string[] { - return (result.diagnostics ?? []).map((d) => `${d.code}: ${d.message}`); - } - test("islands only the patching module, with the builtin-patch reason", async () => { const dir = copyCorpus("auto"); const result = await buildDir(dir, "main"); @@ -725,6 +728,30 @@ console.log(reader.payload.trim(), result.text); expect(blocked.islands?.packages ?? []).toEqual([]); }, 300_000); + test("a native read of the patched member through destructuring is SC8006", async () => { + // A destructured read is an entry read like a direct `fs.readFileSync` + // one: the entry stays native, so it cannot join the island set and + // would not see the patch. + const dir = copyCorpus( + "read-destructured", + `import { createRequire } from "node:module"; +import { loadPayload } from "./loader.ts"; +const require = createRequire(import.meta.url); +const { readFileSync } = require("fs") as typeof import("fs"); +const result = await loadPayload(); +console.log(result.text, readFileSync("payload.txt", "utf8")); +`, + ); + const result = await buildDir(dir, "main"); + expect(result.ok).toBe(false); + const read = messages(result).find( + (m) => m.startsWith("SC8006:") && m.includes("the entry module reads 'fs.readFileSync'"), + ); + expect(read).toBeDefined(); + expect(read).toContain("fs.readFileSync"); + expect(read).toContain("loader.ts"); + }, 300_000); + test("--node-islands=off makes the patch a native refusal again", async () => { const dir = copyCorpus( "off", @@ -864,6 +891,320 @@ export function patchAndBump(): number { rmSync(islandCache, { recursive: true, force: true }); } }, 300_000); + + // The module tree refuses a program module whose loads it cannot trace + // (they run in Node from the shipped tree, where nothing else provides + // them). Collection runs before any statement fence, so the refusal has + // to be a diagnostic — never a thrown PoisonError reaching the CLI. + test("a load the island tree cannot trace is SC8006", async () => { + const dir = copyCorpus( + "computed", + `import { loadPayload } from "./loader.ts"; +console.log(await loadPayload("payload-reader")); +`, + { + "loader.ts": `import { createRequire } from "node:module"; +const require = createRequire(import.meta.url); +const fs = require("fs") as typeof import("fs"); +fs.readFileSync = (() => "") as typeof fs.readFileSync; +export async function loadPayload(name: string): Promise { + return import(name); +} +`, + }, + ); + const result = await buildDir(dir, "main"); + expect(result.ok).toBe(false); + const untraced = messages(result).find( + (m) => m.startsWith("SC8006:") && m.includes("string-literal specifiers only"), + ); + expect(untraced).toBeDefined(); + expect(untraced).toContain("loader.ts"); + expect(untraced).toContain("import() with a computed specifier"); + }); + }); + + // A module that needs the JavaScript runtime itself — it extends a host + // class the program only declares — runs in a Node island automatically + // (R1.6). The entry stays native. + describe("a module that needs a host class", () => { + const globals = `interface HostSocket { + readonly readyState: number; + close(code?: number, reason?: string): void; +} + +declare var WebSocket: { + new (url: string): HostSocket; + readonly prototype: HostSocket; +}; +`; + const socket = `/// +export class Socket extends WebSocket { + static tag = "socket"; +} + +export function describeSocket(): string { + return \`\${Socket.tag}:\${Object.getPrototypeOf(Socket) === WebSocket}\`; +} +`; + const main = `import { describeSocket } from "./socket.ts"; + +console.log("socket:", describeSocket()); +`; + + async function buildProgram(): Promise { + const dir = mkdtempSync(join(tmpdir(), "scriptc-host-class-")); + writeFileSync( + join(dir, "package.json"), + '{ "name": "host-class", "private": true, "type": "module" }\n', + ); + writeFileSync(join(dir, "globals.d.ts"), globals); + writeFileSync(join(dir, "socket.ts"), socket); + writeFileSync(join(dir, "main.ts"), main); + return dir; + } + + test("islands the module and runs it in Node", async () => { + const dir = await buildProgram(); + const result = await withFreshBuildCache(() => + compile(join(dir, "main.ts"), { + outPath: join(dir, "main"), + outDir: dir, + sanitize, + libnode: pack!, + }), + ); + expect(messages(result)).toEqual([]); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.islands?.modules.map((file) => basename(file))).toEqual(["socket.ts"]); + const note = (result.warnings ?? []).find( + (d) => d.code === "SC8001" && d.message.includes("socket.ts"), + ); + expect(note?.message).toContain("Node island"); + // The island's module cache holds the TypeScript module, and Node + // refuses to strip types under node_modules, where this suite keeps + // its shared cache: this run gets a cache of its own. + const executed = await run(join(dir, "main"), [], { + ...process.env, + XDG_CACHE_HOME: join(dir, "cache"), + }); + expect(executed).toEqual({ stdout: "socket: socket:true\n", stderr: "", code: 0 }); + }, 600_000); + + // Node runs TypeScript by erasing types in place: a module that needs a + // transform cannot run in an island, so its refusal stays the build's + // answer. + test("a host class in a file Node cannot strip stays refused", async () => { + const dir = await buildProgram(); + writeFileSync( + join(dir, "socket.ts"), + `/// +enum Mode { + Socket, +} +export class Socket extends WebSocket { + static tag = Mode.Socket === 0 ? "socket" : "other"; +} + +export function describeSocket(): string { + return \`\${Socket.tag}:\${Object.getPrototypeOf(Socket) === WebSocket}\`; +} +`, + ); + const result = await withFreshBuildCache(() => + compile(join(dir, "main.ts"), { outPath: join(dir, "main"), outDir: dir, sanitize }), + ); + expect(result.ok).toBe(false); + expect(messages(result).some((m) => m.startsWith("SC1090: extending classes"))).toBe(true); + }, 600_000); + }); + + // A Worker a module of the program starts from inside a Node island: its + // entry is a module of the program too, so the module tree ships the entry + // and what it imports as real sources, and the worker thread evaluates its + // own copies (Node gives every thread its own module instances). The corpus + // program 6901 pins the behavior against Node; these cases read what the + // build ships and what the island refuses. + describe("a worker entry a Node island starts", () => { + const corpus = join(repoRoot, "tests/corpus/6901-island-worker-entry"); + + /** A scratch copy of the corpus outside node_modules: the worker entry + * ships as TypeScript and Node refuses to strip types below one. */ + function copyCorpus(name: string, extra: Record = {}): string { + const dir = mkdtempSync(join(tmpdir(), `scriptc-island-worker-${name}-`)); + cpSync(corpus, dir, { recursive: true }); + for (const [file, text] of Object.entries(extra)) writeFileSync(join(dir, file), text); + return dir; + } + + async function buildDir(dir: string, libnode: string = pack!) { + return withFreshBuildCache(() => + compile(join(dir, "main.ts"), { + outPath: join(dir, "main"), + outDir: dir, + sanitize, + libnode, + }), + ); + } + + test("ships the entry and what it imports as real sources", async () => { + const dir = copyCorpus("tree"); + const result = await buildDir(dir, "beside"); + expect(messages(result)).toEqual([]); + expect(result.ok).toBe(true); + if (!result.ok) return; + const modules = join(dir, "main.libnode/modules"); + const shipped = (file: string) => readFileSync(join(modules, file), "utf8"); + expect(shipped("worker.ts")).toContain("parentPort?.postMessage"); + // state.ts is the program's own module: the program compiles its copy + // for its own thread, and the worker thread reads this source. + expect(shipped("state.ts")).toContain("export let count = 0;"); + expect(shipped("launcher.ts")).toContain("new Worker(new URL"); + }, 600_000); + + // state.ts is compiled for the program's thread and also runs in the + // worker's thread from the tree: a Worker it starts is the worker + // thread's own, so its entry ships too. + test("ships the worker entries of a compiled module a worker imports", async () => { + const dir = copyCorpus("nested", { + "state.ts": `import { Worker } from "node:worker_threads"; + +export let count = 0; + +export function bump(): number { + count++; + return count; +} + +export function startNested(): Worker { + return new Worker(new URL("./nested.ts", import.meta.url)); +} +`, + "nested.ts": `import { parentPort } from "node:worker_threads"; + +parentPort?.postMessage("nested"); +`, + }); + const result = await buildDir(dir, "beside"); + expect(messages(result)).toEqual([]); + expect(result.ok).toBe(true); + if (!result.ok) return; + const modules = join(dir, "main.libnode/modules"); + expect(readFileSync(join(modules, "nested.ts"), "utf8")).toContain('postMessage("nested")'); + }, 600_000); + + test("runs the worker on its own module instances", async () => { + const dir = copyCorpus("run"); + const result = await buildDir(dir); + expect(messages(result)).toEqual([]); + expect(result.ok).toBe(true); + if (!result.ok) return; + // The island's module cache cannot live under node_modules, where this + // suite keeps its shared one. + const executed = await run(join(dir, "main"), [], { + ...process.env, + XDG_CACHE_HOME: join(dir, "cache"), + }); + expect(executed).toEqual({ + stdout: + "socket: socket:true\nmain: 1 2 2\nworker: echo=21 worker=1,2,3 count=3 echo=22 worker=4,5,6 count=6\nmain after: 2\n", + stderr: "", + code: 0, + }); + }, 600_000); + + // A .mts module is an ES module whatever its package.json says, so the + // island loads it as one: a default import is its default export. + test("an islanded .mts module in a CommonJS package keeps its default export", async () => { + const dir = copyCorpus("mts", { + "package.json": '{ "name": "island-mts", "private": true }\n', + "launcher.mts": `/// +export class Socket extends WebSocket { + static tag = "socket"; +} + +export default function greet(name: string): string { + return \`hello \${name} from \${Socket.tag}:\${Object.getPrototypeOf(Socket) === WebSocket}\`; +} +`, + "main.ts": `import greet from "./launcher.mts"; + +console.log(greet("island")); +`, + }); + const result = await buildDir(dir); + expect(messages(result)).toEqual([]); + expect(result.ok).toBe(true); + if (!result.ok) return; + const executed = await run(join(dir, "main"), [], { + ...process.env, + XDG_CACHE_HOME: join(dir, "cache"), + }); + expect(executed).toEqual({ + stdout: "hello island from socket:true\n", + stderr: "", + code: 0, + }); + }, 600_000); + + // The tree keeps the program's relative layout and no other: an entry the + // build cannot name is refused instead of shipped. + test("a computed entry is SC8006", async () => { + const dir = copyCorpus("computed", { + "launcher.ts": `/// +import { Worker } from "node:worker_threads"; + +export class Socket extends WebSocket { + static tag = "socket"; +} + +export function socketTag(): string { + return \`\${Socket.tag}:\${Object.getPrototypeOf(Socket) === WebSocket}\`; +} + +export function start(name: string): number { + return new Worker(new URL(name, import.meta.url)).threadId; +} +`, + "main.ts": `import { socketTag, start } from "./launcher.ts"; + +console.log(socketTag(), "started", start("worker.ts")); +`, + }); + const result = await buildDir(dir); + expect(result.ok).toBe(false); + const refused = messages(result).find( + (m) => + m.startsWith("SC8006:") && + m.includes("starts a Worker whose entry the build cannot resolve"), + ); + expect(refused).toBeDefined(); + expect(refused).toContain("launcher.ts"); + expect(refused).toContain('new URL("./entry.ts", import.meta.url)'); + }, 600_000); + + test("a worker entry Node cannot strip stays refused", async () => { + const dir = copyCorpus("enum", { + "worker.ts": `import { parentPort } from "node:worker_threads"; + +enum Kind { + A, +} + +parentPort?.postMessage(Kind.A); +`, + }); + const result = await buildDir(dir); + expect(result.ok).toBe(false); + const stripped = messages(result).find( + (m) => m.startsWith("SC8006:") && m.includes("Node cannot strip its types"), + ); + expect(stripped).toBeDefined(); + expect(stripped).toContain("worker.ts"); + expect(stripped).toContain("enum"); + }, 600_000); }); // The build-time probe evaluates modules to decide their deferral. A diff --git a/tests/harness/npm-suite.ts b/tests/harness/npm-suite.ts index daf31e8e6ed..83418bf2a43 100644 --- a/tests/harness/npm-suite.ts +++ b/tests/harness/npm-suite.ts @@ -121,7 +121,10 @@ const KNOWN: Record = { "node24-url-factories": { run: "SC2020" }, "node24-url-mutation": { run: "SC1090" }, "node24-util-adapters": { run: "Object.create with this value is not supported yet" }, - "node24-util-types": { run: "SC1090" }, + // The fixture builds a function from a string, so the package runs in a + // Node island: with a libnode pack the program is compared against Node, + // and without one the build stops with SC8005. + "node24-util-types": { island: true }, "late-island-refusal": { island: true }, "package-imports": { island: true }, "promise-bridge": { build: "SC1090" }, @@ -129,8 +132,11 @@ const KNOWN: Record = { "util-shims": { run: "SC2020" }, // The package redeclares the var it exported: Node copies the first value. "var-export-redeclared": { island: true }, - // `class Upper extends TransformStream`: extending a builtin class from JavaScript. - "web-streams": { run: "SC1090" }, + // `class Upper extends TransformStream`: extending a builtin class from + // JavaScript. The package is an island, and the worker-entry scan ships the + // modules it starts, so the program builds and runs: with a libnode pack it + // is compared against Node, and without one the build stops with SC8005. + "web-streams": { island: true }, "workspace-copied": { island: true }, // The SYMLINKED twin of workspace-copied: the automatic island policy // places the refused package from its workspace realpath.