diff --git a/pyproject.toml b/pyproject.toml index a7d7688..fdef641 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "sphinx-autoapi", "sphinx-copybutton", "sphinx-design", + "sphinx-markdown-builder>=0.6.9", "sphinxcontrib-mermaid>=2.0.0rc0,<2.2", "sphinx-reredirects", "toml", @@ -58,9 +59,6 @@ wiki = [ "sphinx-markdown-builder>=0.6.9", ] -llms = [ - "sphinx-markdown-builder>=0.6.9", -] themes = [ "shibuya", "sphinxawesome-theme", @@ -84,7 +82,6 @@ develop = [ "breathe>=4.35.0", "sphinx-rust", "sphinx-js>=5.0.0", - "sphinx-markdown-builder>=0.6.9", # Themes "shibuya", "sphinxawesome-theme", diff --git a/yardang/cli.py b/yardang/cli.py index 644d00a..5ff0000 100644 --- a/yardang/cli.py +++ b/yardang/cli.py @@ -35,8 +35,6 @@ def build( config_base: str | None = "tool.yardang", previous_versions: bool | None = False, ): - llms_config_base = f"{config_base or 'tool.yardang'}.llms" - use_llms = get_config(section="enabled", base=llms_config_base) is True with generate_docs_configuration( project=project, title=title, @@ -57,25 +55,21 @@ def build( config_base=config_base, previous_versions=previous_versions, ) as file: - build_commands = [[executable, "-m", "sphinx", ".", output, "-c", file]] - if use_llms: - build_commands.append([executable, "-m", "sphinx", "-b", "yardang-llms", ".", output, "-c", file]) - - for build_cmd in build_commands: - if debug: - print(" ".join(build_cmd)) - if quiet: - process = Popen(build_cmd) - else: - process = Popen(build_cmd, stderr=stderr, stdout=stdout) - while process.poll() is None: - sleep(0.1) - if process.returncode != 0: - if pdb: - import pdb # noqa: T100 - - pdb.set_trace() # noqa: T100 - raise Exit(process.returncode) + build_cmd = [executable, "-m", "sphinx", ".", output, "-c", file] + if debug: + print(" ".join(build_cmd)) + if quiet: + process = Popen(build_cmd) + else: + process = Popen(build_cmd, stderr=stderr, stdout=stdout) + while process.poll() is None: + sleep(0.1) + if process.returncode != 0: + if pdb: + import pdb # noqa: T100 + + pdb.set_trace() # noqa: T100 + raise Exit(process.returncode) def debug(): diff --git a/yardang/sphinx/llms.py b/yardang/sphinx/llms.py index 822ab15..0686e4f 100644 --- a/yardang/sphinx/llms.py +++ b/yardang/sphinx/llms.py @@ -1,13 +1,60 @@ from pathlib import Path +from urllib.parse import urlsplit, urlunsplit from docutils import nodes +from sphinx.application import Sphinx from sphinx_markdown_builder.builder import MarkdownBuilder +from sphinx_markdown_builder.translator import MarkdownTranslator from yardang import __version__ +_DOCNAMES_ATTRIBUTE = "_yardang_llms_docnames" + + +def _target_uri(docname: str) -> str: + return f"{docname}.html.md" + + +class _LlmsMarkdownTranslator(MarkdownTranslator): + def visit_toctree(self, node: nodes.Node) -> None: + raise nodes.SkipNode + + def _fetch_ref_uri(self, node: nodes.Node) -> str: + refuri = super()._fetch_ref_uri(node) + parsed = urlsplit(refuri) + if parsed.scheme or parsed.netloc or not parsed.path.endswith(".html"): + return refuri + return urlunsplit(parsed._replace(path=f"{parsed.path}.md")) + + +class _MarkdownRenderer: + """Render resolved Sphinx doctrees without running a second builder.""" + + name = "yardang-llms" + format = "markdown" + default_translator_class = _LlmsMarkdownTranslator + + def __init__(self, app: Sphinx): + self.app = app + self.config = app.config + self.current_doc_name = "" + + def create_translator(self, *args): + return self.app.registry.create_translator(self, *args) + + @staticmethod + def get_target_uri(docname: str, typ: str | None = None) -> str: + return _target_uri(docname) + + def render(self, docname: str, doctree: nodes.document) -> str: + self.current_doc_name = docname + translator = self.create_translator(doctree, self) + doctree.walkabout(translator) + return translator.astext() + class LlmsBuilder(MarkdownBuilder): - """Build LLM-friendly Markdown and index files.""" + """Retain an explicit builder for direct Sphinx use.""" name = "yardang-llms" epilog = "The LLM-friendly documentation is in %(outdir)s." @@ -17,83 +64,127 @@ def init(self): self.out_suffix = ".html.md" def get_outdated_docs(self): - for docname in self._ordered_docnames(): + for docname in self.env.collect_relations(): source_mtime = self._get_source_mtime(docname) target_mtime = self._get_target_mtime(docname) if docname not in self.env.all_docs or source_mtime is None or target_mtime is None or source_mtime > target_mtime: yield docname - def get_target_uri(self, docname: str, typ: str | None = None): - return f"{docname}{self.out_suffix}" - - def finish(self): - docnames = self._ordered_docnames() - self._write_sitemap(docnames) - - full_path = Path(self.outdir) / "llms-full.txt" - if self.config.yardang_llms_full_build: - content = [] - for docname in docnames: - target = Path(self.outdir) / self.get_target_uri(docname) - content.append(f"\n\n{target.read_text(encoding='utf-8').strip()}") - full_path.write_text("\n\n".join(content) + "\n", encoding="utf-8") - else: - full_path.unlink(missing_ok=True) - - def _ordered_docnames(self) -> list[str]: - return list(self.env.collect_relations()) - - def _write_sitemap(self, docnames: list[str]) -> None: - lines = [f"# {self.config.yardang_llms_title}", ""] - description = self.config.yardang_llms_description.strip() - if description: - lines.extend(f"> {line}" for line in description.splitlines()) - lines.append("") - - lines.extend(["## Pages", ""]) - for docname in docnames: - lines.append(f"- [{self._title(docname)}]({self.get_target_uri(docname)}): {self._description(docname)}") - - if self.config.yardang_llms_full_build: - lines.extend(["", "## Full documentation", "", "- [llms-full.txt](llms-full.txt): All pages in one document."]) - - (Path(self.outdir) / "llms.txt").write_text("\n".join(lines) + "\n", encoding="utf-8") - - def _title(self, docname: str) -> str: - if docname == self.config.root_doc: - return self.config.yardang_llms_title - title = self.env.titles.get(docname) - return title.astext() if title is not None else docname.rsplit("/", 1)[-1].replace("_", " ").title() - - def _description(self, docname: str) -> str: - metadata_description = self.env.metadata.get(docname, {}).get("description") - if metadata_description: - return self._shorten(metadata_description) - - doctree = self.env.get_doctree(docname) - for node in doctree.findall(nodes.meta): - if node.get("name") == "description" and node.get("content"): - return self._shorten(node["content"]) - for node in doctree.findall(nodes.paragraph): - if any(node.findall(nodes.image)): - continue - if text := node.astext().strip(): - return self._shorten(text) - return "Documentation page." + def get_target_uri(self, docname: str, typ: str | None = None) -> str: + return _target_uri(docname) + + def finish(self) -> None: + docnames = list(self.env.collect_relations()) + _write_sitemap(self.app, docnames) + _write_full_document(self.app, docnames) + + +def _prepare_markdown(app: Sphinx, builder) -> None: + if builder.name == "html": + setattr(app, _DOCNAMES_ATTRIBUTE, frozenset(app.env.collect_relations())) + + +def _write_markdown(app: Sphinx, doctree: nodes.document, docname: str) -> None: + docnames = getattr(app, _DOCNAMES_ATTRIBUTE, None) + if docnames is None: + docnames = frozenset(app.env.collect_relations()) + setattr(app, _DOCNAMES_ATTRIBUTE, docnames) + if app.builder.name != "html" or docname not in docnames: + return + + renderer = _MarkdownRenderer(app) + target = Path(app.outdir) / renderer.get_target_uri(docname) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(renderer.render(docname, doctree), encoding="utf-8") + + +def _build_llms(app: Sphinx, exception: Exception | None) -> None: + if exception is not None or app.builder.name != "html": + return + + renderer = _MarkdownRenderer(app) + docnames = list(app.env.collect_relations()) + for docname in docnames: + target = Path(app.outdir) / renderer.get_target_uri(docname) + if target.is_file(): + continue + doctree = app.env.get_and_resolve_doctree(docname, app.builder) + if not target.is_file(): + _write_markdown(app, doctree, docname) + _write_sitemap(app, docnames) + _write_full_document(app, docnames) + + +def _write_sitemap(app: Sphinx, docnames: list[str]) -> None: + lines = [f"# {app.config.yardang_llms_title}", ""] + description = app.config.yardang_llms_description.strip() + if description: + lines.extend(f"> {line}" for line in description.splitlines()) + lines.append("") + + lines.extend(["## Pages", ""]) + for docname in docnames: + lines.append(f"- [{_title(app, docname)}]({_target_uri(docname)}): {_description(app, docname)}") + + if app.config.yardang_llms_full_build: + lines.extend(["", "## Full documentation", "", "- [llms-full.txt](llms-full.txt): All pages in one document."]) + + (Path(app.outdir) / "llms.txt").write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def _write_full_document(app: Sphinx, docnames: list[str]) -> None: + full_path = Path(app.outdir) / "llms-full.txt" + if not app.config.yardang_llms_full_build: + full_path.unlink(missing_ok=True) + return + + content = [] + for docname in docnames: + uri = _target_uri(docname) + page = (Path(app.outdir) / uri).read_text(encoding="utf-8").strip() + content.append(f"\n\n{page}") + full_path.write_text("\n\n".join(content) + "\n", encoding="utf-8") + + +def _title(app: Sphinx, docname: str) -> str: + if docname == app.config.root_doc: + return app.config.yardang_llms_title + title = app.env.titles.get(docname) + return title.astext() if title is not None else docname.rsplit("/", 1)[-1].replace("_", " ").title() + + +def _description(app: Sphinx, docname: str) -> str: + metadata_description = app.env.metadata.get(docname, {}).get("description") + if metadata_description: + return _shorten(metadata_description) + + doctree = app.env.get_doctree(docname) + for node in doctree.findall(nodes.meta): + if node.get("name") == "description" and node.get("content"): + return _shorten(node["content"]) + for node in doctree.findall(nodes.paragraph): + if any(node.findall(nodes.image)): + continue + if text := node.astext().strip(): + return _shorten(text) + return "Documentation page." - @staticmethod - def _shorten(value: str, limit: int = 160) -> str: - text = " ".join(value.split()) - return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..." + +def _shorten(value: str, limit: int = 160) -> str: + text = " ".join(value.split()) + return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..." -def setup(app) -> dict[str, object]: - """Register Yardang's LLM-friendly Markdown builder.""" +def setup(app: Sphinx) -> dict[str, object]: + """Generate LLM-friendly files after a successful HTML build.""" app.setup_extension("sphinx_markdown_builder") app.add_config_value("yardang_llms_title", "Documentation", "") app.add_config_value("yardang_llms_description", "", "") app.add_config_value("yardang_llms_full_build", True, "") app.add_builder(LlmsBuilder) + app.connect("write-started", _prepare_markdown) + app.connect("doctree-resolved", _write_markdown, priority=900) + app.connect("build-finished", _build_llms) return { "version": __version__, "parallel_read_safe": True, diff --git a/yardang/tests/test_llms.py b/yardang/tests/test_llms.py index 7829892..ff1ee29 100644 --- a/yardang/tests/test_llms.py +++ b/yardang/tests/test_llms.py @@ -1,5 +1,6 @@ import re from pathlib import Path +from subprocess import Popen from yardang.build import generate_docs_configuration from yardang.cli import build @@ -24,8 +25,8 @@ def _write_project(tmp_path: Path, *, full_build: bool = True) -> None: full-build = {str(full_build).lower()} """ ) - (tmp_path / "README.md").write_text("# Test Project\n\nProject overview.\n") - (tmp_path / "guide.md").write_text("# Guide\n\nGuide summary for language models.\n") + (tmp_path / "README.md").write_text("# Test Project\n\nProject overview.\n\nSee the [guide](guide.md).\n") + (tmp_path / "guide.md").write_text("# Guide\n\nGuide summary for language models.\n\nReturn to the [overview](index.md).\n") (tmp_path / "orphan.md").write_text("# Orphan\n\nThis page is not in the toctree.\n") @@ -44,19 +45,29 @@ def test_generated_configuration_enables_yardang_llms(tmp_path, monkeypatch): assert not (tmp_path / "conf.py").exists() -def test_build_generates_llms_outputs(tmp_path, monkeypatch): +def test_build_generates_llms_outputs_in_one_sphinx_process(tmp_path, monkeypatch): _write_project(tmp_path) monkeypatch.chdir(tmp_path) output = tmp_path / "html" + commands = [] + + def tracked_popen(command, *args, **kwargs): + commands.append(command) + return Popen(command, *args, **kwargs) + + monkeypatch.setattr("yardang.cli.Popen", tracked_popen) build(quiet=True, output=str(output)) + assert len(commands) == 1 assert (output / "index.html").is_file() assert (output / "guide.html").is_file() assert (output / "index.html.md").is_file() assert (output / "guide.html.md").is_file() assert not (output / "orphan.html.md").exists() assert (output / "llms-full.txt").is_file() + assert "(guide.html.md)" in (output / "index.html.md").read_text() + assert "(index.html.md)" in (output / "guide.html.md").read_text() sitemap = (output / "llms.txt").read_text() assert sitemap.startswith("# Test Project\n\n> A project for LLMs\n")