diff --git a/docs/src/configuration.md b/docs/src/configuration.md index 35cf106..73295a5 100644 --- a/docs/src/configuration.md +++ b/docs/src/configuration.md @@ -890,3 +890,32 @@ The generated wiki includes: - `_Sidebar.md` - Navigation sidebar with links to all pages - `_Footer.md` - Footer with links to documentation and repository - All documentation pages converted to GitHub-flavored markdown + +## LLM-friendly Documentation + +The `[tool.yardang.llms]` section controls LLM-friendly output generated by `yardang build`. + +```toml +[tool.yardang.llms] +enabled = true +description = "Documentation for My Project" +full-build = true +``` + +When enabled, the HTML output directory also contains: + +- `llms.txt`: Project description and links to pages reachable from the root toctree. +- `llms-full.txt`: Combined content of all linked pages when `full-build` is enabled. +- `*.html.md`: Markdown representation of each linked HTML page. + +### `enabled` + +Enables LLM-friendly output. Defaults to `false`. + +### `description` + +Sets the project summary in `llms.txt`. Defaults to the project description. + +### `full-build` + +Generates `llms-full.txt` and links it from `llms.txt`. Defaults to `true`. diff --git a/pyproject.toml b/pyproject.toml index 51d77a5..3420729 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ wiki = [ ] llms = [ - "sphinx-llm>=0.4.1", + "sphinx-markdown-builder>=0.6.9", ] themes = [ "shibuya", @@ -85,7 +85,6 @@ develop = [ "sphinx-rust", "sphinx-js>=5.0.0", "sphinx-markdown-builder>=0.6.9", - "sphinx-llm>=0.4.1", # Themes "shibuya", "sphinxawesome-theme", @@ -229,6 +228,4 @@ markdown-flavor = "github" [tool.yardang.llms] enabled = true description = "Easily generate sphinx documentation" -build-parallel = true -suffix-mode = "auto" full-build = true diff --git a/yardang/build.py b/yardang/build.py index b6e203c..b5189cb 100644 --- a/yardang/build.py +++ b/yardang/build.py @@ -525,24 +525,15 @@ def customize(args): # Determine if wiki/markdown output should be generated use_wiki = wiki_args["wiki_enabled"] - # Load sphinx-llm (llms.txt) configuration from tool.yardang.llms + # Load LLM-friendly documentation configuration from tool.yardang.llms llms_config_base = f"{config_base}.llms" llms_args = {} for config_option, default in { - # yardang gate "llms_enabled": False, - # sphinx-llm passthrough options - "llms_txt_description": "", - "llms_txt_build_parallel": True, - "llms_txt_suffix_mode": "auto", - "llms_txt_full_build": True, + "llms_description": "", + "llms_full_build": True, }.items(): - # config keys in toml use hyphens, not underscores, and drop the - # llms_/llms_txt_ prefix - if config_option.startswith("llms_txt_"): - toml_key = config_option.replace("llms_txt_", "").replace("_", "-") - else: - toml_key = config_option.replace("llms_", "").replace("_", "-") + toml_key = config_option.replace("llms_", "").replace("_", "-") llms_args[config_option] = get_config(section=toml_key, base=llms_config_base) if llms_args[config_option] is None: llms_args[config_option] = default @@ -629,19 +620,8 @@ def customize(args): fp.write("index.md\n") if "index.md" not in pages: Path("index.md").touch(exist_ok=True) - # sphinx-llm starts a nested Sphinx build without forwarding the - # generated configuration directory. Make the same configuration - # available from the source directory for that build. - source_configuration = Path("conf.py") - if use_llms: - source_configuration.write_text(template) - - try: - # yield folder path to sphinx build - yield td - finally: - if use_llms: - source_configuration.unlink(missing_ok=True) + # yield folder path to sphinx build + yield td @contextmanager diff --git a/yardang/cli.py b/yardang/cli.py index 244bb96..644d00a 100644 --- a/yardang/cli.py +++ b/yardang/cli.py @@ -35,6 +35,8 @@ def build( config_base: str | None = "tool.yardang", previous_versions: bool | None = False, ): + llms_config_base = f"{config_base or 'tool.yardang'}.llms" + use_llms = get_config(section="enabled", base=llms_config_base) is True with generate_docs_configuration( project=project, title=title, @@ -55,30 +57,25 @@ def build( config_base=config_base, previous_versions=previous_versions, ) as file: - build_cmd = [ - executable, - "-m", - "sphinx", - ".", - output, - "-c", - file, - ] - - if debug: - print(" ".join(build_cmd)) - if quiet: - process = Popen(build_cmd) - else: - process = Popen(build_cmd, stderr=stderr, stdout=stdout) - while process.poll() is None: - sleep(0.1) - if process.returncode != 0: - if pdb: - import pdb # noqa: T100 - - pdb.set_trace() # noqa: T100 - raise Exit(process.returncode) + build_commands = [[executable, "-m", "sphinx", ".", output, "-c", file]] + if use_llms: + build_commands.append([executable, "-m", "sphinx", "-b", "yardang-llms", ".", output, "-c", file]) + + for build_cmd in build_commands: + if debug: + print(" ".join(build_cmd)) + if quiet: + process = Popen(build_cmd) + else: + process = Popen(build_cmd, stderr=stderr, stdout=stdout) + while process.poll() is None: + sleep(0.1) + if process.returncode != 0: + if pdb: + import pdb # noqa: T100 + + pdb.set_trace() # noqa: T100 + raise Exit(process.returncode) def debug(): diff --git a/yardang/conf.py.j2 b/yardang/conf.py.j2 index c8d2fd3..55c52a7 100644 --- a/yardang/conf.py.j2 +++ b/yardang/conf.py.j2 @@ -95,9 +95,9 @@ if use_sphinx_js: if use_wiki: extensions.append("sphinx_markdown_builder") -# Add sphinx-llm extension if llms.txt generation is enabled +# Add Yardang's LLM-friendly documentation builder if enabled if use_llms: - extensions.append("sphinx_llm.txt") + extensions.append("yardang.sphinx.llms") if use_autoapi in (True, None): # add if it is set to true or if it is set to None @@ -253,14 +253,11 @@ if use_wiki: markdown_bullet = "{{markdown_bullet}}" markdown_flavor = "{{markdown_flavor}}" -# sphinx-llm (llms.txt) configuration +# LLM-friendly documentation configuration if use_llms: - {% if llms_txt_description %} - llms_txt_description = """{{llms_txt_description}}""" - {% endif %} - llms_txt_build_parallel = {{llms_txt_build_parallel}} - llms_txt_suffix_mode = "{{llms_txt_suffix_mode}}" - llms_txt_full_build = {{llms_txt_full_build}} + yardang_llms_title = """{{title}}""" + yardang_llms_description = """{{llms_description or description}}""" + yardang_llms_full_build = {{llms_full_build}} # autosummary autosummary_generate = True # if using autosummary, autogenerate diff --git a/yardang/sphinx/llms.py b/yardang/sphinx/llms.py new file mode 100644 index 0000000..822ab15 --- /dev/null +++ b/yardang/sphinx/llms.py @@ -0,0 +1,101 @@ +from pathlib import Path + +from docutils import nodes +from sphinx_markdown_builder.builder import MarkdownBuilder + +from yardang import __version__ + + +class LlmsBuilder(MarkdownBuilder): + """Build LLM-friendly Markdown and index files.""" + + name = "yardang-llms" + epilog = "The LLM-friendly documentation is in %(outdir)s." + + def init(self): + super().init() + self.out_suffix = ".html.md" + + def get_outdated_docs(self): + for docname in self._ordered_docnames(): + source_mtime = self._get_source_mtime(docname) + target_mtime = self._get_target_mtime(docname) + if docname not in self.env.all_docs or source_mtime is None or target_mtime is None or source_mtime > target_mtime: + yield docname + + def get_target_uri(self, docname: str, typ: str | None = None): + return f"{docname}{self.out_suffix}" + + def finish(self): + docnames = self._ordered_docnames() + self._write_sitemap(docnames) + + full_path = Path(self.outdir) / "llms-full.txt" + if self.config.yardang_llms_full_build: + content = [] + for docname in docnames: + target = Path(self.outdir) / self.get_target_uri(docname) + content.append(f"\n\n{target.read_text(encoding='utf-8').strip()}") + full_path.write_text("\n\n".join(content) + "\n", encoding="utf-8") + else: + full_path.unlink(missing_ok=True) + + def _ordered_docnames(self) -> list[str]: + return list(self.env.collect_relations()) + + def _write_sitemap(self, docnames: list[str]) -> None: + lines = [f"# {self.config.yardang_llms_title}", ""] + description = self.config.yardang_llms_description.strip() + if description: + lines.extend(f"> {line}" for line in description.splitlines()) + lines.append("") + + lines.extend(["## Pages", ""]) + for docname in docnames: + lines.append(f"- [{self._title(docname)}]({self.get_target_uri(docname)}): {self._description(docname)}") + + if self.config.yardang_llms_full_build: + lines.extend(["", "## Full documentation", "", "- [llms-full.txt](llms-full.txt): All pages in one document."]) + + (Path(self.outdir) / "llms.txt").write_text("\n".join(lines) + "\n", encoding="utf-8") + + def _title(self, docname: str) -> str: + if docname == self.config.root_doc: + return self.config.yardang_llms_title + title = self.env.titles.get(docname) + return title.astext() if title is not None else docname.rsplit("/", 1)[-1].replace("_", " ").title() + + def _description(self, docname: str) -> str: + metadata_description = self.env.metadata.get(docname, {}).get("description") + if metadata_description: + return self._shorten(metadata_description) + + doctree = self.env.get_doctree(docname) + for node in doctree.findall(nodes.meta): + if node.get("name") == "description" and node.get("content"): + return self._shorten(node["content"]) + for node in doctree.findall(nodes.paragraph): + if any(node.findall(nodes.image)): + continue + if text := node.astext().strip(): + return self._shorten(text) + return "Documentation page." + + @staticmethod + def _shorten(value: str, limit: int = 160) -> str: + text = " ".join(value.split()) + return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..." + + +def setup(app) -> dict[str, object]: + """Register Yardang's LLM-friendly Markdown builder.""" + app.setup_extension("sphinx_markdown_builder") + app.add_config_value("yardang_llms_title", "Documentation", "") + app.add_config_value("yardang_llms_description", "", "") + app.add_config_value("yardang_llms_full_build", True, "") + app.add_builder(LlmsBuilder) + return { + "version": __version__, + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/yardang/tests/test_llms.py b/yardang/tests/test_llms.py index cec9260..7829892 100644 --- a/yardang/tests/test_llms.py +++ b/yardang/tests/test_llms.py @@ -1,15 +1,13 @@ -"""Tests for llms.txt generation (sphinx-llm) in yardang.""" - -import os +import re from pathlib import Path +from yardang.build import generate_docs_configuration +from yardang.cli import build -class TestLlmsConfiguration: - """Tests for llms.txt configuration loading and generation.""" - def test_llms_config_loading_from_pyproject(self, tmp_path): - """Test that llms configuration is loaded from pyproject.toml.""" - pyproject_content = """ +def _write_project(tmp_path: Path, *, full_build: bool = True) -> None: + (tmp_path / "pyproject.toml").write_text( + f""" [project] name = "test-project" version = "1.0.0" @@ -17,133 +15,83 @@ def test_llms_config_loading_from_pyproject(self, tmp_path): [tool.yardang] title = "Test Project" root = "README.md" +pages = ["guide.md"] use-autoapi = false [tool.yardang.llms] enabled = true description = "A project for LLMs" -build-parallel = false -suffix-mode = "replace" -full-build = false +full-build = {str(full_build).lower()} """ - pyproject_path = tmp_path / "pyproject.toml" - pyproject_path.write_text(pyproject_content) - - readme_path = tmp_path / "README.md" - readme_path.write_text("# Test Project\n\nTest content.") + ) + (tmp_path / "README.md").write_text("# Test Project\n\nProject overview.\n") + (tmp_path / "guide.md").write_text("# Guide\n\nGuide summary for language models.\n") + (tmp_path / "orphan.md").write_text("# Orphan\n\nThis page is not in the toctree.\n") - original_cwd = os.getcwd() - try: - os.chdir(tmp_path) - from yardang.utils import get_config +def test_generated_configuration_enables_yardang_llms(tmp_path, monkeypatch): + _write_project(tmp_path) + monkeypatch.chdir(tmp_path) - assert get_config(section="enabled", base="tool.yardang.llms") is True - assert get_config(section="description", base="tool.yardang.llms") == "A project for LLMs" - assert get_config(section="build-parallel", base="tool.yardang.llms") is False - assert get_config(section="suffix-mode", base="tool.yardang.llms") == "replace" - assert get_config(section="full-build", base="tool.yardang.llms") is False - finally: - os.chdir(original_cwd) + with generate_docs_configuration() as conf_dir: + conf_content = (Path(conf_dir) / "conf.py").read_text() - def test_llms_config_defaults(self, tmp_path): - """Test that llms configuration returns None when not specified.""" - pyproject_content = """ -[project] -name = "test-project" -version = "1.0.0" - -[tool.yardang] -title = "Test Project" -root = "README.md" -""" - pyproject_path = tmp_path / "pyproject.toml" - pyproject_path.write_text(pyproject_content) - - readme_path = tmp_path / "README.md" - readme_path.write_text("# Test Project\n\nTest content.") - - original_cwd = os.getcwd() - try: - os.chdir(tmp_path) - - from yardang.utils import get_config - - assert get_config(section="enabled", base="tool.yardang.llms") is None - assert get_config(section="suffix-mode", base="tool.yardang.llms") is None - finally: - os.chdir(original_cwd) - - def test_generate_docs_with_llms_enabled(self, tmp_path): - """Test that generate_docs_configuration wires in the sphinx-llm extension.""" - pyproject_content = """ -[project] -name = "test-project" -version = "1.0.0" + assert "use_llms = True" in conf_content + assert 'extensions.append("yardang.sphinx.llms")' in conf_content + assert 'yardang_llms_description = """A project for LLMs"""' in conf_content + assert "yardang_llms_full_build = True" in conf_content + assert "sphinx_llm" not in conf_content + assert not (tmp_path / "conf.py").exists() -[tool.yardang] -title = "Test Project" -root = "README.md" -use-autoapi = false -[tool.yardang.llms] -enabled = true -description = "A project for LLMs" -suffix-mode = "replace" -""" - pyproject_path = tmp_path / "pyproject.toml" - pyproject_path.write_text(pyproject_content) +def test_build_generates_llms_outputs(tmp_path, monkeypatch): + _write_project(tmp_path) + monkeypatch.chdir(tmp_path) + output = tmp_path / "html" - readme_path = tmp_path / "README.md" - readme_path.write_text("# Test Project\n\nTest content.") + build(quiet=True, output=str(output)) - original_cwd = os.getcwd() - try: - os.chdir(tmp_path) + assert (output / "index.html").is_file() + assert (output / "guide.html").is_file() + assert (output / "index.html.md").is_file() + assert (output / "guide.html.md").is_file() + assert not (output / "orphan.html.md").exists() + assert (output / "llms-full.txt").is_file() - from yardang.build import generate_docs_configuration + sitemap = (output / "llms.txt").read_text() + assert sitemap.startswith("# Test Project\n\n> A project for LLMs\n") + assert "- [Test Project](index.html.md): Project overview." in sitemap + assert "- [Guide](guide.html.md): Guide summary for language models." in sitemap + assert "Orphan" not in sitemap + assert "[llms-full.txt](llms-full.txt)" in sitemap - with generate_docs_configuration() as conf_dir: - conf_content = (Path(conf_dir) / "conf.py").read_text() + for target in re.findall(r"\]\(([^)]+)\)", sitemap): + assert (output / target).is_file() - assert "use_llms = True" in conf_content - assert 'extensions.append("sphinx_llm.txt")' in conf_content - assert "llms_txt_suffix_mode" in conf_content - assert "replace" in conf_content - assert "A project for LLMs" in conf_content - assert Path("conf.py").read_text() == conf_content - assert not Path("conf.py").exists() - finally: - os.chdir(original_cwd) +def test_full_build_can_be_disabled(tmp_path, monkeypatch): + _write_project(tmp_path) + monkeypatch.chdir(tmp_path) + output = tmp_path / "html" - def test_generate_docs_llms_disabled_by_default(self, tmp_path): - """Test that llms.txt generation is off unless explicitly enabled.""" - pyproject_content = """ -[project] -name = "test-project" -version = "1.0.0" + build(quiet=True, output=str(output)) + assert (output / "llms-full.txt").is_file() -[tool.yardang] -title = "Test Project" -root = "README.md" -use-autoapi = false -""" - pyproject_path = tmp_path / "pyproject.toml" - pyproject_path.write_text(pyproject_content) + _write_project(tmp_path, full_build=False) + build(quiet=True, output=str(output)) - readme_path = tmp_path / "README.md" - readme_path.write_text("# Test Project\n\nTest content.") + assert (output / "llms.txt").is_file() + assert not (output / "llms-full.txt").exists() + assert "llms-full.txt" not in (output / "llms.txt").read_text() - original_cwd = os.getcwd() - try: - os.chdir(tmp_path) - from yardang.build import generate_docs_configuration +def test_llms_generation_is_disabled_by_default(tmp_path, monkeypatch): + (tmp_path / "pyproject.toml").write_text('[project]\nname = "test-project"\nversion = "1.0.0"\n\n[tool.yardang]\nuse-autoapi = false\n') + (tmp_path / "README.md").write_text("# Test Project\n") + monkeypatch.chdir(tmp_path) + output = tmp_path / "html" - with generate_docs_configuration() as conf_dir: - conf_content = (Path(conf_dir) / "conf.py").read_text() + build(quiet=True, output=str(output)) - assert "use_llms = False" in conf_content - finally: - os.chdir(original_cwd) + assert (output / "index.html").is_file() + assert not (output / "llms.txt").exists()