Replace sphinx-llm with Yardang builder

docs/src/configuration.md

@@ -890,3 +890,32 @@ The generated wiki includes:
 - `_Sidebar.md` - Navigation sidebar with links to all pages
 - `_Footer.md` - Footer with links to documentation and repository
 - All documentation pages converted to GitHub-flavored markdown
+
+## LLM-friendly Documentation
+
+The `[tool.yardang.llms]` section controls LLM-friendly output generated by `yardang build`.
+
+```toml
+[tool.yardang.llms]
+enabled = true
+description = "Documentation for My Project"
+full-build = true
+```
+
+When enabled, the HTML output directory also contains:
+
+- `llms.txt`: Project description and links to pages reachable from the root toctree.
+- `llms-full.txt`: Combined content of all linked pages when `full-build` is enabled.
+- `*.html.md`: Markdown representation of each linked HTML page.
+
+### `enabled`
+
+Enables LLM-friendly output. Defaults to `false`.
+
+### `description`
+
+Sets the project summary in `llms.txt`. Defaults to the project description.
+
+### `full-build`
+
+Generates `llms-full.txt` and links it from `llms.txt`. Defaults to `true`.

pyproject.toml

@@ -59,7 +59,7 @@ wiki = [
 
 ]
 llms = [
-    "sphinx-llm>=0.4.1",
+    "sphinx-markdown-builder>=0.6.9",
 ]
 themes = [
     "shibuya",
@@ -85,7 +85,6 @@ develop = [
     "sphinx-rust",
     "sphinx-js>=5.0.0",
     "sphinx-markdown-builder>=0.6.9",
-    "sphinx-llm>=0.4.1",
     # Themes
     "shibuya",
     "sphinxawesome-theme",
@@ -229,6 +228,4 @@ markdown-flavor = "github"
 [tool.yardang.llms]
 enabled = true
 description = "Easily generate sphinx documentation"
-build-parallel = true
-suffix-mode = "auto"
 full-build = true

yardang/build.py

@@ -525,24 +525,15 @@ def customize(args):
         # Determine if wiki/markdown output should be generated
         use_wiki = wiki_args["wiki_enabled"]
 
-        # Load sphinx-llm (llms.txt) configuration from tool.yardang.llms
+        # Load LLM-friendly documentation configuration from tool.yardang.llms
         llms_config_base = f"{config_base}.llms"
         llms_args = {}
         for config_option, default in {
-            # yardang gate
             "llms_enabled": False,
-            # sphinx-llm passthrough options
-            "llms_txt_description": "",
-            "llms_txt_build_parallel": True,
-            "llms_txt_suffix_mode": "auto",
-            "llms_txt_full_build": True,
+            "llms_description": "",
+            "llms_full_build": True,
         }.items():
-            # config keys in toml use hyphens, not underscores, and drop the
-            # llms_/llms_txt_ prefix
-            if config_option.startswith("llms_txt_"):
-                toml_key = config_option.replace("llms_txt_", "").replace("_", "-")
-            else:
-                toml_key = config_option.replace("llms_", "").replace("_", "-")
+            toml_key = config_option.replace("llms_", "").replace("_", "-")
             llms_args[config_option] = get_config(section=toml_key, base=llms_config_base)
             if llms_args[config_option] is None:
                 llms_args[config_option] = default
@@ -629,19 +620,8 @@ def customize(args):
                             fp.write("index.md\n")
             if "index.md" not in pages:
                 Path("index.md").touch(exist_ok=True)
-            # sphinx-llm starts a nested Sphinx build without forwarding the
-            # generated configuration directory. Make the same configuration
-            # available from the source directory for that build.
-            source_configuration = Path("conf.py")
-            if use_llms:
-                source_configuration.write_text(template)
-
-            try:
-                # yield folder path to sphinx build
-                yield td
-            finally:
-                if use_llms:
-                    source_configuration.unlink(missing_ok=True)
+            # yield folder path to sphinx build
+            yield td
 
 
 @contextmanager

yardang/cli.py

@@ -35,6 +35,8 @@ def build(
     config_base: str | None = "tool.yardang",
     previous_versions: bool | None = False,
 ):
+    llms_config_base = f"{config_base or 'tool.yardang'}.llms"
+    use_llms = get_config(section="enabled", base=llms_config_base) is True
     with generate_docs_configuration(
         project=project,
         title=title,
@@ -55,30 +57,25 @@ def build(
         config_base=config_base,
         previous_versions=previous_versions,
     ) as file:
-        build_cmd = [
-            executable,
-            "-m",
-            "sphinx",
-            ".",
-            output,
-            "-c",
-            file,
-        ]
-
-        if debug:
-            print(" ".join(build_cmd))
-        if quiet:
-            process = Popen(build_cmd)
-        else:
-            process = Popen(build_cmd, stderr=stderr, stdout=stdout)
-        while process.poll() is None:
-            sleep(0.1)
-        if process.returncode != 0:
-            if pdb:
-                import pdb  # noqa: T100
-
-                pdb.set_trace()  # noqa: T100
-            raise Exit(process.returncode)
+        build_commands = [[executable, "-m", "sphinx", ".", output, "-c", file]]
+        if use_llms:
+            build_commands.append([executable, "-m", "sphinx", "-b", "yardang-llms", ".", output, "-c", file])
+
+        for build_cmd in build_commands:
+            if debug:
+                print(" ".join(build_cmd))
+            if quiet:
+                process = Popen(build_cmd)
+            else:
+                process = Popen(build_cmd, stderr=stderr, stdout=stdout)
+            while process.poll() is None:
+                sleep(0.1)
+            if process.returncode != 0:
+                if pdb:
+                    import pdb  # noqa: T100
+
+                    pdb.set_trace()  # noqa: T100
+                raise Exit(process.returncode)
 
 
 def debug():

yardang/conf.py.j2

@@ -95,9 +95,9 @@ if use_sphinx_js:
 if use_wiki:
     extensions.append("sphinx_markdown_builder")
 
-# Add sphinx-llm extension if llms.txt generation is enabled
+# Add Yardang's LLM-friendly documentation builder if enabled
 if use_llms:
-    extensions.append("sphinx_llm.txt")
+    extensions.append("yardang.sphinx.llms")
 
 if use_autoapi in (True, None):
     # add if it is set to true or if it is set to None
@@ -253,14 +253,11 @@ if use_wiki:
     markdown_bullet = "{{markdown_bullet}}"
     markdown_flavor = "{{markdown_flavor}}"
 
-# sphinx-llm (llms.txt) configuration
+# LLM-friendly documentation configuration
 if use_llms:
-    {% if llms_txt_description %}
-    llms_txt_description = """{{llms_txt_description}}"""
-    {% endif %}
-    llms_txt_build_parallel = {{llms_txt_build_parallel}}
-    llms_txt_suffix_mode = "{{llms_txt_suffix_mode}}"
-    llms_txt_full_build = {{llms_txt_full_build}}
+    yardang_llms_title = """{{title}}"""
+    yardang_llms_description = """{{llms_description or description}}"""
+    yardang_llms_full_build = {{llms_full_build}}
 
 # autosummary
 autosummary_generate = True  # if using autosummary, autogenerate

yardang/sphinx/llms.py

@@ -0,0 +1,101 @@
+from pathlib import Path
+
+from docutils import nodes
+from sphinx_markdown_builder.builder import MarkdownBuilder
+
+from yardang import __version__
+
+
+class LlmsBuilder(MarkdownBuilder):
+    """Build LLM-friendly Markdown and index files."""
+
+    name = "yardang-llms"
+    epilog = "The LLM-friendly documentation is in %(outdir)s."
+
+    def init(self):
+        super().init()
+        self.out_suffix = ".html.md"
+
+    def get_outdated_docs(self):
+        for docname in self._ordered_docnames():
+            source_mtime = self._get_source_mtime(docname)
+            target_mtime = self._get_target_mtime(docname)
+            if docname not in self.env.all_docs or source_mtime is None or target_mtime is None or source_mtime > target_mtime:
+                yield docname
+
+    def get_target_uri(self, docname: str, typ: str | None = None):
+        return f"{docname}{self.out_suffix}"
+
+    def finish(self):
+        docnames = self._ordered_docnames()
+        self._write_sitemap(docnames)
+
+        full_path = Path(self.outdir) / "llms-full.txt"
+        if self.config.yardang_llms_full_build:
+            content = []
+            for docname in docnames:
+                target = Path(self.outdir) / self.get_target_uri(docname)
+                content.append(f"<!-- {self.get_target_uri(docname)} -->\n\n{target.read_text(encoding='utf-8').strip()}")
+            full_path.write_text("\n\n".join(content) + "\n", encoding="utf-8")
+        else:
+            full_path.unlink(missing_ok=True)
+
+    def _ordered_docnames(self) -> list[str]:
+        return list(self.env.collect_relations())
+
+    def _write_sitemap(self, docnames: list[str]) -> None:
+        lines = [f"# {self.config.yardang_llms_title}", ""]
+        description = self.config.yardang_llms_description.strip()
+        if description:
+            lines.extend(f"> {line}" for line in description.splitlines())
+            lines.append("")
+
+        lines.extend(["## Pages", ""])
+        for docname in docnames:
+            lines.append(f"- [{self._title(docname)}]({self.get_target_uri(docname)}): {self._description(docname)}")
+
+        if self.config.yardang_llms_full_build:
+            lines.extend(["", "## Full documentation", "", "- [llms-full.txt](llms-full.txt): All pages in one document."])
+
+        (Path(self.outdir) / "llms.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+    def _title(self, docname: str) -> str:
+        if docname == self.config.root_doc:
+            return self.config.yardang_llms_title
+        title = self.env.titles.get(docname)
+        return title.astext() if title is not None else docname.rsplit("/", 1)[-1].replace("_", " ").title()
+
+    def _description(self, docname: str) -> str:
+        metadata_description = self.env.metadata.get(docname, {}).get("description")
+        if metadata_description:
+            return self._shorten(metadata_description)
+
+        doctree = self.env.get_doctree(docname)
+        for node in doctree.findall(nodes.meta):
+            if node.get("name") == "description" and node.get("content"):
+                return self._shorten(node["content"])
+        for node in doctree.findall(nodes.paragraph):
+            if any(node.findall(nodes.image)):
+                continue
+            if text := node.astext().strip():
+                return self._shorten(text)
+        return "Documentation page."
+
+    @staticmethod
+    def _shorten(value: str, limit: int = 160) -> str:
+        text = " ".join(value.split())
+        return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..."
+
+
+def setup(app) -> dict[str, object]:
+    """Register Yardang's LLM-friendly Markdown builder."""
+    app.setup_extension("sphinx_markdown_builder")
+    app.add_config_value("yardang_llms_title", "Documentation", "")
+    app.add_config_value("yardang_llms_description", "", "")
+    app.add_config_value("yardang_llms_full_build", True, "")
+    app.add_builder(LlmsBuilder)
+    return {
+        "version": __version__,
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }