Name island reasons, run host-class modules in Node islands, and ship Worker entries into the island tree

docs/content/limitations/native-worker-threads.mdx

@@ -1,5 +1,8 @@
 `node:worker_threads` runs workers as native threads.
 Worker entry points must resolve at build time: use a source-relative `new URL("./worker.ts", import.meta.url)`, the current module URL, an absolute path, or CommonJS `__filename`.
+A module that runs in a Node island starts Node's own Worker instead: its entry must be a source-relative `new URL("./worker.ts", import.meta.url)` built from that module's URL, the build ships the entry and the modules it imports beside the island's modules, and the worker thread evaluates its own copies of them.
+The build refuses a Worker whose entry it cannot resolve to such a path, and a TypeScript module of the program that ships this way whose types Node cannot strip in a worker thread.
+A TypeScript module the worker's entry imports from outside the program ships unchecked, and Node reports its own error there if it cannot strip it.
 Each worker initializes its own module state, event loop, environment snapshot, and reference-counted heap.
 Nested workers inherit their parent's environment snapshot.
 Supported constructor options are `workerData`, `argv`, a `transferList` of MessagePorts, `name` (reported by `threadName` and `worker.threadName`), `stdout`, `stderr` and `stdin`, and `resourceLimits.stackSizeMb`, which sets the worker thread's stack size (at least 192 KiB; without it a worker's stack is 8 MiB, where Node's is 4 MiB).

packages/compiler/src/executable/prepare.ts

@@ -52,7 +52,7 @@ import {
   isIslandReasonCode,
 } from "../node-islands/classify.js";
 import { packageDirOf, packageVersionOf } from "../node-islands/origin.js";
-import { describe } from "../node-islands/summary.js";
+import { describe, type NodeIslandReason } from "../node-islands/summary.js";
 import { resolveBareModule } from "../frontend/resolve.js";
 import type { NodeIslandScan } from "../node-islands/scan.js";
 import { growIslandSet, type IslandGrowthContext } from "../node-islands/growth.js";
@@ -248,7 +248,13 @@ export function prepareExecutableModule(
       // Sloppy user modules the lowering refused join the same retry
       // (DESIGN.md note 1 §3.2): one reload with every island at once,
       // never one round per module.
-      const grownModules = loweringModules(entryPath, opts, lowered.diagnostics, lowerModules);
+      const grownModules = loweringModules(
+        entryPath,
+        opts,
+        lowered.diagnostics,
+        lowerModules,
+        (files) => fe.nodeIslandScan?.({ userJs: false, stripFor: files }) ?? null,
+      );
       if (grown !== null || grownModules !== null) {
         fe.dispose();
         frontendDisposed();
@@ -450,6 +456,14 @@ const SLOPPY_REASONS: ReadonlySet<string> = new Set([
  * builtin, so it runs in Node, where an island consumer sees it. */
 const PATCH_REASONS: ReadonlySet<string> = new Set(["builtin-patch"]);
 
+/** The runtime reasons a TypeScript module turns into an island module for:
+ * the JavaScript runtime owns the construct's base class, so the module
+ * needs nothing shipped beside it. A load only the runtime can resolve (a
+ * computed specifier, a worker entry path) stays refused — the shipped tree
+ * cannot carry what the module loads, and an island would fail where the
+ * native build ran. */
+const ISLAND_RUN_REASONS: ReadonlySet<string> = new Set(["host-class-base"]);
+
 /** The program modules that run in Node islands, by file: modules marked
  * with `--node-island=<path>` or `scriptc.nodeIslands.modules`, and (under
  * `--node-islands=auto`) user JavaScript the pre-scan found a sloppy-only
@@ -1055,28 +1069,59 @@ function loweringModules(
   opts: CompileRequestOptions,
   diagnostics: readonly ScrDiagnostic[],
   known: ReadonlyMap<string, IslandModule>,
+  scanForRun: (files: readonly string[]) => NodeIslandScan | null,
 ): Map<string, IslandModule> | null {
   const policy = nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands);
   if (policy.mode !== "auto" || known.size > 16) return null;
   const grown = new Map(known);
+  // TypeScript modules the refusals name, with the reason that names them
+  // (a module joins once, with its first reason).
+  const runtimeReasons = new Map<string, NodeIslandReason>();
   for (const d of diagnostics) {
     if (!isIslandReasonCode(d.code)) continue;
     const file = d.loc.file;
     // Packages run in Node through their own decision
     // (node-islands/growth.ts).
     if (file.includes("/node_modules/") || packageDirOf(file) !== null) continue;
-    if (!isJsSourceFileName(file) || sameFile(file, entryPath)) continue;
-    if (grown.has(file)) continue;
+    if (sameFile(file, entryPath) || grown.has(file) || runtimeReasons.has(file)) continue;
     const reason = classifyDiagnostic(d, "refusal");
     // A gap is a construct scriptc does not implement, not a sloppy
     // semantic: running it in Node would hide a compiler limitation.
     if (reason.class === "gap") continue;
-    grown.set(file, {
-      file,
-      class: reason.class,
-      reason: describe(reason),
-      decision: "auto",
-    });
+    if (isJsSourceFileName(file)) {
+      grown.set(file, {
+        file,
+        class: reason.class,
+        reason: describe(reason),
+        decision: "auto",
+      });
+      continue;
+    }
+    // A TypeScript module runs in Node only for a need of the JavaScript
+    // runtime itself that the shipped tree satisfies (ISLAND_RUN_REASONS):
+    // Node then does what the native pipeline could not. An object-model
+    // need is a JavaScript module's sloppy semantics, and a refusal caused
+    // by another declaration is that declaration's cascade.
+    if (!ISLAND_RUN_REASONS.has(reason.reason) || d.cause !== undefined) continue;
+    if (!/\.(?:ts|mts|cts)$/.test(file) || /\.d\.[cm]?ts$/.test(file)) continue;
+    runtimeReasons.set(file, reason);
+  }
+  if (runtimeReasons.size > 0) {
+    const scan = scanForRun([...runtimeReasons.keys()]);
+    for (const [file, reason] of runtimeReasons) {
+      const module = scan?.modules.find((m) => m.file === file);
+      // Node runs TypeScript by erasing types in place: a module that needs
+      // a transform, or resolves a relative import only without its
+      // extension, cannot run in an island as written, and its refusal
+      // stays the build's answer.
+      if (module === undefined || module.stripProblem !== null) continue;
+      grown.set(file, {
+        file,
+        class: reason.class,
+        reason: describe(reason),
+        decision: "auto",
+      });
+    }
   }
   return grown.size === known.size ? null : grown;
 }

packages/compiler/src/frontend/fork-target.ts

@@ -301,6 +301,57 @@ function staticString(
   return null;
 }
 
+/** Whether an entry expression names its file relative to the module that
+ * starts it: `import.meta.url`, `__filename`, or a URL whose base is one of
+ * those (`new URL("./entry.ts", import.meta.url)`, through consts,
+ * templates and fileURLToPath). That relation is the one a shipped Node
+ * island tree keeps, because every collected file keeps its layout
+ * relative to one root; an absolute path or a `file:` URL built from
+ * anything else names the build machine, which the tree does not hold. */
+export function fromModuleUrl(
+  program: ts.Program,
+  expr: ts.Expression,
+  seen: Set<ts.Symbol> = new Set(),
+): boolean {
+  const current = strip(expr);
+  if (importMetaUrl(current)) return true;
+  if (ts.isIdentifier(current) && current.text === "__filename")
+    return isNodeGlobal(program, current);
+  const init = constInitializer(program, current);
+  if (init !== null && ts.isIdentifier(current)) {
+    const symbol = program.getTypeChecker().getSymbolAtLocation(current);
+    if (symbol === undefined || seen.has(symbol)) return false;
+    seen.add(symbol);
+    const value = fromModuleUrl(program, init, seen);
+    seen.delete(symbol);
+    return value;
+  }
+  if (
+    ts.isCallExpression(current) &&
+    !current.questionDotToken &&
+    current.arguments.nodes.length === 1 &&
+    ts.isIdentifier(current.expression) &&
+    isBuiltinMemberImport(program, current.expression, "url", "fileURLToPath")
+  )
+    return fromModuleUrl(program, current.arguments.nodes[0]!, seen);
+  if (
+    ts.isNewExpression(current) &&
+    !current.questionDotToken &&
+    ts.isIdentifier(current.expression) &&
+    (isBuiltinMemberImport(program, current.expression, "url", "URL") ||
+      (current.expression.text === "URL" && isNodeGlobal(program, current.expression)))
+  ) {
+    const args = current.arguments?.nodes ?? [];
+    if (args.length !== 2) return false;
+    const input = staticString(program, args[0]!, seen);
+    // A reference relative to the base: an absolute one would not move
+    // with the module.
+    if (input === null || input.startsWith("/") || /^[a-z][a-z0-9+.-]*:/i.test(input)) return false;
+    return fromModuleUrl(program, args[1]!, seen);
+  }
+  return false;
+}
+
 /** Resolves the supported fork modulePath expression to an absolute source
  * path. Plain runtime strings are intentionally excluded: worker selection
  * is part of the compiled graph, not a runtime filesystem lookup. */

packages/compiler/src/frontend/island-worker-entry.ts

@@ -0,0 +1,135 @@
+/* Worker entries of a module whose shipped copy Node runs (scope-B: an
+ * island module, or a module such an island loads). Node starts a worker
+ * thread with its own module instances: the thread re-evaluates the entry
+ * and everything the entry imports. So the module tree ships the entry and
+ * its static import subtree as real sources (node-islands/modules.ts), and
+ * the Worker keeps its path: the URL the module builds from its own URL at
+ * build time names the same file in the tree.
+ *
+ * Only a form the tree can keep is accepted: an entry relative to the
+ * module's own URL (`new URL("./entry.ts", import.meta.url)`, through
+ * consts, templates and fileURLToPath; `import.meta.url` itself;
+ * `__filename`). Every other form a build can see is refused, because the
+ * tree keeps relative layout and nothing else:
+ *  - a computed or unbounded entry: the build cannot know what to ship;
+ *  - an absolute path or a `file:` URL that is not built from the module's
+ *    own URL: the tree never holds a build-machine path;
+ *  - a path relative to the process's working directory (Node resolves a
+ *    plain relative filename against `cwd`): the tree is not the cwd.
+ * A static entry that names no file at all (a data: URL, a package
+ * specifier, a missing path) is Node's own business and stays as it is.
+ *
+ * A TypeScript file that ships this way must pass stripProblem: Node
+ * erases types in place in a worker thread too, and it refuses to do so
+ * inside node_modules.
+ *
+ * The scan reads only the program's own sources, and never the TypeScript
+ * parser: a worker entry is a program root and its import graph is in the
+ * program (worker-target.ts), so the files this ships are exactly the ones
+ * the checker bound. A module of the tree the program does not hold has no
+ * checked source, and its Worker stays Node's own business, as it was before
+ * the tree carried it. The compiler itself is compiled natively, where the
+ * parser island's rpc-api/rpc-process modules cannot lower — importing them
+ * from a lowering-side module breaks the self-hosted build. */
+import { resolve } from "node:path";
+import { trackedFileExists } from "./input-tracker.js";
+import * as ts from "./ts7/adapter.js";
+import { tsgoPath } from "./dts-paths.js";
+import { fromModuleUrl, staticForkString } from "./fork-target.js";
+import { moduleSourceCandidates } from "./module-source-candidates.js";
+import { isWorkerConstructor, workerModulePath } from "./worker-target.js";
+import type { NodeIslandWorkerScan } from "../node-islands/modules.js";
+import { stripProblem } from "../node-islands/type-strip.js";
+
+export interface IslandWorkerScan {
+  /** The worker entries `file`'s shipped copy starts, or the reason one
+   * cannot ship (node-islands/modules.ts asks for each traced module).
+   * `fresh`: the file belongs to a worker entry's subtree, so its shipped
+   * copy runs in the worker thread even when the program compiles it. */
+  workers(file: string, fresh: boolean): NodeIslandWorkerScan;
+  /** Why Node cannot run `file`'s source as shipped, or null. */
+  sourceProblem(file: string): string | null;
+}
+
+const TYPESCRIPT = /\.[mc]?ts$/;
+
+const NONE: NodeIslandWorkerScan = { kind: "entries", files: [] };
+
+/** The scan over one program: `compiled` answers whether the program lowers
+ * a file natively, whose Worker constructions the native lowering answers
+ * (lowerWorkerNew), so outside a worker entry's subtree they never run from
+ * the shipped tree. Inside one (`fresh`), the worker thread evaluates the
+ * file's shipped copy, natively compiled or not, and its Worker
+ * constructions are scanned. `compiled` is asked on every call, not read
+ * once: the module order grows while lowering (worker targets join it) and
+ * the tree is collected after. */
+export function createIslandWorkerScan(
+  program: ts.Program,
+  compiled: (file: string) => boolean,
+): IslandWorkerScan {
+  let programSources: Map<string, ts.SourceFile> | null = null;
+  const key = (file: string): string => tsgoPath(resolve(file));
+  const sourceOf = (file: string): ts.SourceFile | null => {
+    programSources ??= new Map();
+    if (programSources.size === 0)
+      for (const sf of program.getImplementationSourceFiles())
+        programSources.set(key(sf.fileName), sf);
+    return programSources.get(key(file)) ?? null;
+  };
+
+  const workers = (file: string, fresh: boolean): NodeIslandWorkerScan => {
+    if (!fresh && compiled(key(file))) return NONE;
+    // A file outside the program has no checked source: its Worker cannot be
+    // recognised, and Node reports its entry as a missing module in the
+    // worker thread exactly as it did before the tree carried the file.
+    const sf = sourceOf(file);
+    if (sf === null) return NONE;
+    const entries = new Set<string>();
+    for (const construction of moduleSourceCandidates(program, sf).constructions) {
+      const args = construction.arguments?.nodes ?? [];
+      // An arity the constructor rejects is the runtime's error, in the
+      // island exactly as under Node.
+      if (args.length === 0 || args.length > 2 || args.some(ts.isSpreadElement)) continue;
+      if (!isWorkerConstructor(program, construction.expression)) continue;
+      const entry = args[0]!;
+      const path = workerModulePath(program, entry);
+      if (path === null) {
+        // A static entry that names no file of the build (a data: URL a
+        // package specifier, a filename relative to the working directory)
+        // is Node's to resolve; only an entry the build cannot even see is
+        // refused.
+        if (staticForkString(program, entry) !== null) continue;
+        return { kind: "refused", reason: refusal(sf, construction, "cannot resolve") };
+      }
+      if (!fromModuleUrl(program, entry))
+        return {
+          kind: "refused",
+          reason: refusal(sf, construction, `names the build-machine path '${path}'`),
+        };
+      // A file that does not exist is loaded (and fails) exactly as under
+      // Node: there is nothing for the tree to carry.
+      if (trackedFileExists(path)) entries.add(path);
+    }
+    return entries.size === 0 ? NONE : { kind: "entries", files: [...entries].sort() };
+  };
+
+  const sourceProblem = (file: string): string | null => {
+    if (!TYPESCRIPT.test(file)) return null;
+    if (/[\\/]node_modules[\\/]/.test(file))
+      return `'${file}' runs in a worker thread of a Node island, but Node refuses to strip types inside node_modules`;
+    const sf = sourceOf(file);
+    if (sf === null) return null;
+    const problem = stripProblem(program, sf);
+    if (problem === null) return null;
+    return `'${file}' runs in a worker thread of a Node island, but Node cannot strip its types: ${problem.construct}`;
+  };
+
+  return { workers, sourceProblem };
+}
+
+/** A one-line description of the construction, anchored at its source: the
+ * collector's diagnostics carry the reason as a message. */
+function refusal(sf: ts.SourceFile, node: ts.Node, what: string): string {
+  const line = sf.text.slice(0, ts.tokenStartOfNode(node)).split("\n").length;
+  return `${sf.fileName}:${line} starts a Worker whose entry the build ${what}; a worker entry that runs in a Node island is new URL("./entry.ts", import.meta.url), so the build ships the entry and the modules it imports beside the module`;
+}

packages/compiler/src/frontend/lowering/lower-modules.ts

@@ -661,6 +661,37 @@ export function collectProgram(lowerer: Lowerer, parts: FileParts[]): void {
   markRequireEarlyBindings(lowerer);
 }
 
+/** collectIslandImport inside the collection pass. A module the island's
+ * shipped tree cannot carry (a computed load, an unshippable resolution)
+ * has its SC8006 pushed and then throws PoisonError, which a statement
+ * fence around a body would catch. Collection runs outside those fences, so
+ * the throw is caught here: the diagnostic the loader pushed is the build's
+ * answer, and the import stays an island load of the module. */
+function collectIslandImportAtCollection(
+  lowerer: Lowerer,
+  sf: ts.SourceFile,
+  stmt: ts.Statement,
+  spec: string,
+): boolean {
+  try {
+    return collectIslandImport(lowerer, sf, stmt, spec);
+  } catch (error) {
+    if (!(error instanceof PoisonError)) throw error;
+    return true;
+  }
+}
+
+/** collectIslandRequires inside the collection pass, for the same reason as
+ * collectIslandImportAtCollection: a module the shipped tree cannot carry
+ * reports its SC8006 and throws, and the diagnostic is the build's answer. */
+function collectIslandRequiresAtCollection(lowerer: Lowerer, sf: ts.SourceFile): void {
+  try {
+    collectIslandRequires(lowerer, sf);
+  } catch (error) {
+    if (!(error instanceof PoisonError)) throw error;
+  }
+}
+
 export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void {
   const entryPackageFile = entryPackageFilePredicate(lowerer.entry.fileName);
   for (const fp of parts) {
@@ -724,14 +755,19 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void {
       // A relative import of a program module the pre-scan put in a Node
       // island (DESIGN.md note 1 §3.2): the island loads at this module's
       // position and its bindings are island values.
-      if (lowerer.nodeIslands !== null && collectIslandImport(lowerer, fp.sf, stmt, spec)) continue;
+      if (
+        lowerer.nodeIslands !== null &&
+        collectIslandImportAtCollection(lowerer, fp.sf, stmt, spec)
+      )
+        continue;
       const projectDep = isRelativeSpecifier(spec)
         ? null
         : resolveImport(lowerer.program, fp.sf, spec);
       if (projectDep !== null) {
         // A Node island package that tsconfig `paths` maps to its installed
         // directory resolves to its declarations: it loads in Node.
-        if (projectDep.isDeclarationFile) collectIslandImport(lowerer, fp.sf, stmt, spec);
+        if (projectDep.isDeclarationFile)
+          collectIslandImportAtCollection(lowerer, fp.sf, stmt, spec);
         continue;
       }
       const npm = resolveNpmImport(fp.sf.fileName, spec);
@@ -785,7 +821,7 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void {
       if (npm !== null && probeNodeImportRefusal(fp.sf.fileName, spec) !== null) continue;
       // A Node island package (scope-B NI-4): loaded in Node at this
       // module's position, its bindings are island values.
-      if (collectIslandImport(lowerer, fp.sf, stmt, spec)) continue;
+      if (collectIslandImportAtCollection(lowerer, fp.sf, stmt, spec)) continue;
       {
         lowerer.pushDiag(unsupportedPackageImportDiag(npm?.packageName ?? relPkg!, locOf(stmt)));
         continue;
@@ -794,7 +830,7 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void {
   }
   for (const fp of parts) {
     collectDynamicImports(lowerer, fp.sf);
-    collectIslandRequires(lowerer, fp.sf);
+    collectIslandRequiresAtCollection(lowerer, fp.sf);
   }
 }