feat(extract): opt-in --html-as-code to route .html through the AST path (#1230)

README.md

@@ -633,6 +633,7 @@ graphify-out/
 /graphify ./raw                    # run on a specific folder
 /graphify ./raw --mode deep        # more aggressive relationship extraction
 graphify extract ./raw --code-only # index code only — local AST, no API key (skips docs/PDFs/images); an `extract` flag, not a skill flag
+graphify extract ./raw --html-as-code # treat .html as code (embedded <script> JS AST) instead of a semantic document; opt-in, persists for update/watch
 /graphify ./raw --update           # re-extract only changed files
 /graphify ./raw --directed         # preserve edge direction
 /graphify ./raw --cluster-only     # rerun clustering on existing graph

graphify/cli.py

@@ -2442,7 +2442,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
             print(
                 "Usage: graphify extract <path> [--backend gemini|kimi|claude|openai|deepseek|ollama] "
                 "[--model M] [--mode deep] [--out DIR|--output DIR] [--google-workspace] [--no-cluster] "
-                "[--no-gitignore] [--code-only] "
+                "[--no-gitignore] [--code-only] [--html-as-code] "
                 "[--max-workers N] [--token-budget N] [--max-concurrency N] "
                 "[--api-timeout S] [--postgres DSN] [--cargo] [--allow-partial] [--timing]",
                 file=sys.stderr,
@@ -2471,6 +2471,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
         google_workspace = False
         global_merge = False
         code_only = False
+        html_as_code = False
         no_gitignore = False
         global_repo_tag: str | None = None
         # Performance/tuning knobs (issue #792). None means "use library default".
@@ -2538,6 +2539,8 @@ def _parse_float(name: str, raw: str) -> float:
                 dedup_llm = True; i += 1
             elif a == "--code-only":
                 code_only = True; i += 1
+            elif a == "--html-as-code":
+                html_as_code = True; i += 1
             elif a == "--google-workspace":
                 google_workspace = True; i += 1
             elif a == "--no-gitignore":
@@ -2625,6 +2628,7 @@ def _parse_float(name: str, raw: str) -> float:
             _write_build_config as _write_build_cfg,
             _read_build_excludes as _read_build_ex,
             _read_build_gitignore as _read_build_gi,
+            _read_build_html_as_code as _read_build_hac,
         )
         # #1971 persistence: an explicit --no-gitignore persists False; a later
         # flag-less `graphify extract` must NOT clobber it back to True, which
@@ -2636,10 +2640,16 @@ def _parse_float(name: str, raw: str) -> float:
         _effective_gitignore = False if no_gitignore else _read_build_gi(graphify_out)
         # An explicit list replaces the persisted one; omission reuses it.
         _effective_excludes = cli_excludes or _read_build_ex(graphify_out)
+        # Same persistence contract as gitignore: --html-as-code opts .html into
+        # the code path for THIS run and every later update/watch/hook rebuild
+        # (#1230), until a later run explicitly omits the flag on a build whose
+        # config never set it either.
+        _effective_html_as_code = True if html_as_code else _read_build_hac(graphify_out)
         _write_build_cfg(
             graphify_out,
             excludes=cli_excludes or None,
             gitignore=False if no_gitignore else None,
+            html_as_code=True if html_as_code else None,
         )
 
         stages = _StageTimer(cli_timing)
@@ -2691,6 +2701,7 @@ def _parse_float(name: str, raw: str) -> float:
                 google_workspace=google_workspace or None,
                 extra_excludes=_effective_excludes or None,
                 gitignore=_effective_gitignore,
+                html_as_code=_effective_html_as_code,
             )
             files_by_type = detection.get("files", {})
             new_by_type = detection.get("new_files", {})
@@ -2719,6 +2730,7 @@ def _parse_float(name: str, raw: str) -> float:
                 extra_excludes=_effective_excludes or None,
                 cache_root=out_root,
                 gitignore=_effective_gitignore,
+                html_as_code=_effective_html_as_code,
             )
             files_by_type = detection.get("files", {})
             code_files = [Path(p) for p in files_by_type.get("code", [])]

graphify/detect.py

@@ -434,7 +434,7 @@ def _shebang_file_type(path: Path) -> FileType | None:
     return None
 
 
-def classify_file(path: Path) -> FileType | None:
+def classify_file(path: Path, *, html_as_code: bool = False) -> FileType | None:
     # Package manifests (apm.yml, pyproject.toml, go.mod, pom.xml) are parsed
     # deterministically, so route them to the AST path (CODE) rather than the LLM
     # document path — otherwise apm.yml (a .yml "document") would be LLM-extracted
@@ -448,6 +448,16 @@ def classify_file(path: Path) -> FileType | None:
     ext = path.suffix.lower()
     if not ext:
         return _shebang_file_type(path)
+    # Opt-in (#1230): .html is DOCUMENT by default (semantic/LLM path, see below)
+    # because most .html is a template/report, not application logic. Projects
+    # that DO treat their .html as code (e.g. Angular component templates with
+    # meaningful embedded <script> logic) can pass --html-as-code to route it
+    # through the local AST path instead — see extract_html(). Checked before the
+    # CODE_EXTENSIONS/DOC_EXTENSIONS membership below and before the
+    # _looks_like_paper() sniff (an explicit opt-in should not be second-guessed
+    # by the paper heuristic, which exists for prose formats, not markup).
+    if html_as_code and ext == ".html":
+        return FileType.CODE
     if ext in CODE_EXTENSIONS:
         return FileType.CODE
     if ext in PAPER_EXTENSIONS:
@@ -1235,7 +1245,7 @@ def _resolves_under_root(path: Path, root: Path) -> bool:
     return True
 
 
-def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict:
+def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True, html_as_code: bool = False) -> dict:
     root = root.resolve()
     if follow_symlinks is None:
         follow_symlinks = False
@@ -1386,7 +1396,7 @@ def _on_walk_error(err: OSError) -> None:
         if _is_sensitive(p):
             skipped_sensitive.append(str(p))
             continue
-        ftype = classify_file(p)
+        ftype = classify_file(p, html_as_code=html_as_code)
         if not ftype:
             # Considered but unclassifiable: an extension not in any supported set,
             # or an extensionless, non-shebang file (Dockerfile, Gemfile, Makefile,
@@ -1748,6 +1758,7 @@ def detect_incremental(
     kind: str = "semantic",
     extra_excludes: list[str] | None = None,
     gitignore: bool = True,
+    html_as_code: bool = False,
 ) -> dict:
     """Like detect(), but returns only new or modified files since the last run.
 
@@ -1777,6 +1788,7 @@ def detect_incremental(
         google_workspace=google_workspace,
         extra_excludes=extra_excludes,
         gitignore=gitignore,
+        html_as_code=html_as_code,
     )
     # Pass ``root`` so a manifest written with relative keys (post-#777) is
     # re-anchored to the absolute form the rest of this function compares

graphify/extract.py

@@ -77,6 +77,7 @@
     _decldef_class_stem,
     _disambiguate_colliding_node_ids,
     _find_workspace_root,
+    _html_mask_non_script,
     _is_type_like_definition,
     _js_call_identifier,
     _js_default_export_name,
@@ -1587,6 +1588,39 @@ def extract_vue(path: Path) -> dict:
     return result
 
 
+def extract_html(path: Path) -> dict:
+    """Extract functions/classes/imports from JS embedded in a .html file's
+    ``<script>`` blocks (#1230 — opt-in via ``--html-as-code``).
+
+    ``.html`` is DOCUMENT by default (see ``detect.classify_file``): most
+    HTML is a template or report, not application logic, and always shipping
+    it through the local AST path would silently change output for every
+    existing user. ``--html-as-code`` opts a project in; when it does, this
+    is the extractor that runs.
+
+    Mirrors :func:`extract_vue`: masks everything outside inline, JS-typed
+    ``<script>`` bodies (see :func:`_html_mask_non_script` — external
+    ``src="..."`` scripts and non-JS ``type="..."`` blocks are blanked, not
+    parsed) and feeds the result to the plain JS grammar. Plain HTML has no
+    ``lang="ts"`` convention the way a Vue SFC does, so unlike
+    :func:`extract_vue` there is no grammar selection: browsers execute
+    inline ``<script>`` as JavaScript, never TypeScript. Preserved newlines
+    keep ``source_location`` line numbers accurate in the *host* .html file,
+    and multiple ``<script>`` blocks in one file are all covered — each
+    becomes its own masked region, not just the first.
+
+    A file with no ``<script>`` block, or only external/non-JS ones, masks to
+    an all-blank source: `_extract_generic` still emits the file's own node
+    (structure), just with no JS symbol children.
+    """
+    try:
+        src = path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return {"nodes": [], "edges": []}
+    masked = _html_mask_non_script(src)
+    return _extract_generic(path, _JS_CONFIG, source_override=masked.encode("utf-8"))
+
+
 def extract_java(path: Path) -> dict:
     """Extract classes, interfaces, methods, constructors, and imports from a .java file."""
     return _extract_generic(path, _JAVA_CONFIG)
@@ -3979,6 +4013,12 @@ def add_existing_edge(edge: dict) -> None:
     ".vue": extract_vue,
     ".svelte": extract_svelte,
     ".astro": extract_astro,
+    # .html is DOCUMENT by default (detect.DOC_EXTENSIONS) — this entry only
+    # fires when a file is actually dispatched here as code, i.e. when a run
+    # opted in with --html-as-code (#1230). Registering it unconditionally is
+    # safe: extension-keyed dispatch is inert for a path that never enters
+    # code_files in the first place.
+    ".html": extract_html,
     ".dart": extract_dart,
     ".v": extract_verilog,
     ".sv": extract_verilog,

graphify/extractors/resolution.py

@@ -560,6 +560,64 @@ def _blank(s: str) -> str:
     out.append(_blank(src[pos:]))
     return "".join(out), lang
 
+
+# ── .html host + embedded <script> JS (opt-in, --html-as-code, #1230) ─────────
+#
+# Same shape as the <script> half of _VUE_SCRIPT_RE: an open tag (attrs may
+# hold a quoted '>' inside src="...?x=1>2"-style values, hence the alternation
+# instead of a bare [^>]*), a body, a close tag.
+_HTML_SCRIPT_RE = re.compile(
+    r"""(<script\b(?:"[^"]*"|'[^']*'|[^>"'])*>)([\s\S]*?)(</script\s*>)""",
+    re.IGNORECASE,
+)
+
+_HTML_SCRIPT_TYPE_RE = re.compile(r"""\btype\s*=\s*(?:"([^"]*)"|'([^']*)'|(\S+))""", re.IGNORECASE)
+_HTML_SCRIPT_SRC_RE = re.compile(r"""\bsrc\s*=""", re.IGNORECASE)
+
+# type="" values that mean "this is executable JS". Absent type defaults to JS
+# per the HTML spec. Anything else (application/json, text/template,
+# application/ld+json, ...) is data or a client-side template, not JS, and
+# must not be fed to the tree-sitter JS grammar.
+_HTML_SCRIPT_JS_TYPES = frozenset({
+    "", "text/javascript", "application/javascript", "application/x-javascript",
+    "text/ecmascript", "application/ecmascript", "module",
+})
+
+
+def _html_mask_non_script(src: str) -> str:
+    """Blank everything outside inline, JS-typed ``<script>`` bodies.
+
+    Mirrors :func:`_vue_mask_non_script`: markup/attrs/style become spaces so
+    the JS grammar sees only script bodies, while preserved ``\\r``/``\\n``
+    keep line numbers accurate in the *host* .html file. A ``<script
+    src="...">`` (external file — no local JS to parse) and a ``<script
+    type="...">`` naming a non-JS MIME (``application/json``,
+    ``text/template``, ...) are both blanked along with the surrounding
+    markup: their content is not local JS and parsing it as such would only
+    produce noise (a stray ERROR node at best, a wrong node at worst).
+    """
+    def _blank(s: str) -> str:
+        return re.sub(r"[^\r\n]", " ", s)
+
+    out: list[str] = []
+    pos = 0
+    for m in _HTML_SCRIPT_RE.finditer(src):
+        open_tag, body, close_tag = m.group(1), m.group(2), m.group(3)
+        out.append(_blank(src[pos:m.start()]))
+        out.append(_blank(open_tag))
+        type_m = _HTML_SCRIPT_TYPE_RE.search(open_tag)
+        script_type = (type_m.group(1) or type_m.group(2) or type_m.group(3) or "").strip().lower() if type_m else ""
+        is_external = bool(_HTML_SCRIPT_SRC_RE.search(open_tag))
+        if is_external or script_type not in _HTML_SCRIPT_JS_TYPES:
+            out.append(_blank(body))
+        else:
+            out.append(body)
+        out.append(_blank(close_tag))
+        pos = m.end()
+    out.append(_blank(src[pos:]))
+    return "".join(out)
+
+
 def _source_key(source_file: str, root: Path) -> str:
     if not source_file:
         return ""

graphify/watch.py

@@ -81,12 +81,13 @@ def _write_build_config(
     *,
     excludes: "list[str] | None",
     gitignore: bool | None = None,
+    html_as_code: bool | None = None,
 ) -> None:
     """Persist corpus-shaping options under ``out_dir``.
 
     Best effort and non clobbering: omitted options retain their existing values.
     """
-    if not excludes and gitignore is None:
+    if not excludes and gitignore is None and html_as_code is None:
         return
     try:
         out_dir.mkdir(parents=True, exist_ok=True)
@@ -101,6 +102,8 @@ def _write_build_config(
             config["excludes"] = list(excludes)
         if gitignore is not None:
             config["gitignore"] = gitignore
+        if html_as_code is not None:
+            config["html_as_code"] = html_as_code
         path.write_text(json.dumps(config), encoding="utf-8")
     except OSError:
         pass
@@ -133,6 +136,25 @@ def _read_build_gitignore(out_dir: Path) -> bool:
     return True
 
 
+def _read_build_html_as_code(out_dir: Path) -> bool:
+    """Return whether rebuilds should classify .html as code (default False, #1230).
+
+    Mirrors ``_read_build_gitignore``: an ``update``/``watch``/hook rebuild
+    re-runs ``detect()`` from scratch, so the initial ``extract
+    --html-as-code`` opt-in must be persisted or a later rebuild would
+    silently drop .html back to the semantic/document path.
+    """
+    try:
+        path = out_dir / _BUILD_CONFIG_FILENAME
+        if path.is_file():
+            cfg = json.loads(path.read_text(encoding="utf-8"))
+            if isinstance(cfg, dict) and isinstance(cfg.get("html_as_code"), bool):
+                return cfg["html_as_code"]
+    except (OSError, json.JSONDecodeError):
+        pass
+    return False
+
+
 def _merge_changed_paths(*sources: "list[Path] | None") -> list[Path]:
     """Concatenate path lists, preserving order and dropping duplicates.
 
@@ -515,7 +537,18 @@ def _reconcile_existing_graph(
             identity = source_paths.identity(source_file)
             if not source_paths.in_watch_root(source_file):
                 continue
-            if _get_extractor(Path(source_file)) is None:
+            # .html always reads as a semantic source here, regardless of the
+            # current run's --html-as-code state: _get_extractor(".html") is
+            # non-None unconditionally (dispatch is a static extension map),
+            # but an EXISTING .html node in the graph may predate the flag, or
+            # predate this run's flag value, and this per-run detect() cannot
+            # tell which extraction tier produced it. Treating it uniformly as
+            # semantic avoids a spurious "left the scan corpus" warning on
+            # every incremental rebuild of a repo that never opted in (#1230)
+            # — a genuinely stale/rebuilt .html AST node is still evicted
+            # correctly below via current_sources/_origin, which this branch
+            # does not gate.
+            if _get_extractor(Path(source_file)) is None or Path(source_file).suffix.lower() == ".html":
                 # Non-AST source (semantic doc/paper/image — .txt/.pdf/.png/...):
                 # never present in current_sources (built from AST-extractable
                 # code_files), so corpus absence is meaningless. Disk absence is
@@ -937,21 +970,32 @@ def _rebuild_code(
         from graphify.export import to_json, to_html
         from graphify.security import check_graph_file_size_cap
 
-        # Re-apply the excludes the initial extract recorded, so an update/watch/
-        # hook rebuild does not silently re-include deliberately excluded paths
-        # (#1886).
+        # Re-apply the excludes/--html-as-code the initial extract recorded, so
+        # an update/watch/hook rebuild does not silently re-include deliberately
+        # excluded paths (#1886) or drop .html back to the document path (#1230).
         _persisted_excludes = _read_build_excludes(out)
         detected = detect(
             watch_path, follow_symlinks=follow_symlinks,
             extra_excludes=_persisted_excludes or None,
             gitignore=_read_build_gitignore(out),
+            html_as_code=_read_build_html_as_code(out),
         )
         code_files = [Path(f) for f in detected['files']['code']]
 
         # Include document files that have AST extractors (e.g. .md, .mdx, .qmd)
         ast_doc_files: list[Path] = []
         for doc_file in detected['files'].get('document', []):
             p = Path(doc_file)
+            # .html is excluded from this fallback unconditionally: it always has
+            # a registered extractor (extract_html), but is only ever meant to run
+            # through it when --html-as-code opted the project in — and when it
+            # did, detect() above already classified .html as code, so it is in
+            # detected['files']['code'] (and therefore code_files) already, never
+            # in this 'document' list to begin with. This guard is the belt to
+            # that suspenders: it keeps .html out of the AST-fallback path when
+            # the flag is off, matching the opt-in contract (#1230).
+            if p.suffix.lower() == ".html":
+                continue
             if _get_extractor(p) is not None:
                 code_files.append(p)
                 ast_doc_files.append(p)