docs: add RTD doc pages

.github/workflows/tests.yml

@@ -59,6 +59,13 @@ jobs:
       run: |
         python -m pytest --cov=./ --cov-report=xml
 
+    - name: Test docs
+      # version matches .readthedocs.yml
+      if: ${{ matrix.python-version == '3.13' }}
+      run: |
+        uv pip install --system -e ".[docs]"
+        pytest docs/test_docs.py
+
     # coverage
     - name: Upload coverage to Codecov
       if: ${{ matrix.python-version == '3.13' }}

.readthedocs.yml

@@ -0,0 +1,19 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs
+
+sphinx:
+  configuration: docs/source/conf.py

README.md

@@ -4,7 +4,7 @@
 [![Python package](https://img.shields.io/pypi/v/courlan.svg)](https://pypi.python.org/pypi/courlan)
 [![Python versions](https://img.shields.io/pypi/pyversions/courlan.svg)](https://pypi.python.org/pypi/courlan)
 [![Code Coverage](https://img.shields.io/codecov/c/github/adbar/courlan.svg)](https://codecov.io/gh/adbar/courlan)
-[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Documentation](https://readthedocs.org/projects/courlan/badge/?version=latest)](http://courlan.readthedocs.org/en/latest/)
 
 
 ## Why coURLan?

docs/Makefile

@@ -0,0 +1,11 @@
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = docs/source
+BUILDDIR      = docs/_build
+
+.PHONY: html
+html:
+	$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)/html"
+
+.PHONY: clean
+clean:
+	rm -rf $(BUILDDIR)/*

docs/source/api/clean.md

@@ -0,0 +1,24 @@
+# courlan.clean
+
+Core URL cleaning and normalization utilities.
+
+```{automodule} courlan.clean
+:members:
+:undoc-members:
+:show-inheritance:
+```
+
+## Common usage
+
+```python
+from courlan import clean_url, scrub_url, normalize_url, validate_url
+
+# Clean and normalize a URL (returns str or None if invalid)
+url = clean_url('HTTPS://WWW.EXAMPLE.COM:443/path?utm_source=x')
+
+# Basic validation
+is_valid, parsed = validate_url('https://example.com')
+
+# Normalization only
+normalized = normalize_url('http://example.com/path?z=1&a=2#fragment')
+```

docs/source/api/cli.md

@@ -0,0 +1,11 @@
+# courlan.cli
+
+Command-line interface implementation and argument parsing.
+
+This module contains the CLI entry point and internal helpers. Most users interact with this through the `courlan` command rather than importing it directly — see the [CLI Reference](../usage/cli.md) for full flag documentation.
+
+```{automodule} courlan.cli
+:members:
+:undoc-members:
+:show-inheritance:
+```

docs/source/api/core.md

@@ -0,0 +1,38 @@
+# courlan.core
+
+Core URL checking utilities.
+
+```{automodule} courlan.core
+:members:
+:undoc-members:
+:show-inheritance:
+```
+
+## Common usage
+
+```python
+from courlan import check_url, extract_links, filter_links
+
+# check_url returns (url, domain) or None if the URL is rejected
+result = check_url('https://example.com/article')
+if result:
+    url, domain = result
+
+# Strict mode and language filtering
+result = check_url('https://example.com/article', strict=True, language='en')
+
+# Extract links from HTML (returns a set)
+links = extract_links(html, 'https://example.com', external_bool=False)
+
+# Extract and prioritize links for crawling (returns links, priority_links)
+links, priority_links = filter_links(html, 'https://example.com', lang='en')
+```
+
+## Filtering cost
+
+Options add overhead in this order, from cheapest to most expensive:
+
+1. **Basic** — `check_url(url)`
+2. **Language filtering** — `check_url(url, language='en')` — minimal overhead
+3. **Strict mode** — `check_url(url, strict=True)` — more conditions checked
+4. **Redirect checks** — `check_url(url, with_redirects=True)` — network I/O; avoid on large datasets

docs/source/api/filters.md

@@ -0,0 +1,32 @@
+# courlan.filters
+
+URL filtering heuristics for content validation and crawler optimization.
+
+```{automodule} courlan.filters
+:members:
+:undoc-members:
+:show-inheritance:
+```
+
+## Common usage
+
+```python
+from courlan import check_url, filter_links, lang_filter, is_valid_url
+
+# check_url returns (url, domain) or None if rejected
+result = check_url('https://example.com/article', language='en', strict=True)
+if result:
+    url, domain = result
+
+# Extract and filter links from HTML
+html = '<a href="/page1">Link</a><a href="/tag/spam">Tag</a>'
+links, priority_links = filter_links(html, 'https://example.com', lang='en')
+
+# Test if a URL matches a target language heuristically
+if lang_filter('https://example.com/en/article', language='en'):
+    print("Language matches")
+
+# Basic structural validity check (no network call)
+if is_valid_url('https://example.com/path'):
+    print("Valid URL structure")
+```

docs/source/api/index.md

@@ -0,0 +1,20 @@
+# API Reference
+
+This section is generated from the courlan package. Click a module to jump to its reference.
+
+```{toctree}
+:maxdepth: 1
+:caption: Modules
+
+clean
+cli
+core
+filters
+meta
+network
+sampling
+settings
+urlstore
+urlutils
+```
+

docs/source/api/meta.md

@@ -0,0 +1,66 @@
+# courlan.meta
+
+Cache management and meta-utilities.
+
+```{automodule} courlan.meta
+:members:
+:undoc-members:
+:show-inheritance:
+```
+
+## Cache Management
+
+Courlan uses LRU (Least Recently Used) caches to speed up URL parsing and language detection. For long-running processes, you can clear these caches to reclaim memory.
+
+### clear_caches()
+
+**Purpose**: Reset all internal LRU caches.
+
+Use in long-running processes handling many distinct URLs, in memory-constrained environments, or between crawl phases.
+
+**What gets cleared**: urllib.parse results, language detection scores.
+
+**Example**:
+```python
+from courlan import check_url
+from courlan.meta import clear_caches
+
+for i in range(10000):
+    result = check_url(f'https://example.com/page{i}')
+    if (i + 1) % 1000 == 0:
+        clear_caches()
+```
+
+---
+
+## Usage in Batch Workflows
+
+```python
+from courlan import UrlStore, check_url
+from courlan.meta import clear_caches
+
+store = UrlStore(compressed=True)
+store.add_urls(many_urls)
+
+# Process in batches
+batch_size = 5000
+processed = 0
+
+while store.unvisited_websites_number() > 0:
+    for domain in store.get_unvisited_domains():
+        url = store.get_url(domain)
+        if url:
+            check_url(url, strict=True, language='en')
+            processed += 1
+
+            # Clear caches periodically
+            if processed % batch_size == 0:
+                clear_caches()
+                print(f"Processed {processed} URLs, caches cleared")
+```
+
+---
+
+## See Also
+
+- [Web Crawling guide](../usage/crawling.md) — cache clearing in crawler workflows

docs/source/api/network.md

@@ -0,0 +1,19 @@
+# courlan.network
+
+Network helpers for redirect checking and HTTP operations.
+
+```{automodule} courlan.network
+:members:
+:undoc-members:
+:show-inheritance:
+```
+
+## Common usage
+
+```python
+# Redirect checking is typically used through check_url() function
+from courlan import check_url
+
+# Check if URL redirects (makes HTTP HEAD request)
+url, domain = check_url('https://example.com/old-page', with_redirects=True)
+```

docs/source/api/sampling.md

@@ -0,0 +1,74 @@
+# courlan.sampling
+
+Sampling utilities to produce per-host URL samples.
+
+```{automodule} courlan.sampling
+:members:
+:undoc-members:
+:show-inheritance:
+```
+
+## Common usage
+
+```python
+from courlan import sample_urls
+
+# Generate sample: up to 10 URLs per domain
+urls = ['https://example.com/p1', 'https://example.com/p2', 'https://other.org/a']
+sample = sample_urls(urls, 10)
+
+# With exclusion filters
+sample = sample_urls(urls, samplesize=5, exclude_min=2, exclude_max=100)
+```
+
+## Example: Sampling output
+
+**Input URLs** (8 total, 3 domains):
+
+```
+https://github.com/adbar/courlan
+https://github.com/adbar/trafilatura
+https://github.com/adbar/htmldate
+https://example.com/some/page
+https://example.com/another/page
+https://example.com/third/page
+https://another.example/path
+https://another.example/blog/post
+```
+
+**Sample with `samplesize=2`** (2 URLs per domain):
+
+```python
+from courlan import sample_urls
+
+urls = [
+    'https://github.com/adbar/courlan',
+    'https://github.com/adbar/trafilatura',
+    'https://github.com/adbar/htmldate',
+    'https://example.com/some/page',
+    'https://example.com/another/page',
+    'https://example.com/third/page',
+    'https://another.example/path',
+    'https://another.example/blog/post',
+]
+
+sample = sample_urls(urls, samplesize=2)
+# Result: 6 URLs (2 per domain)
+for url in sample:
+    print(url)
+```
+
+**Output**:
+```
+https://github.com/adbar/courlan
+https://github.com/adbar/trafilatura
+https://example.com/some/page
+https://example.com/another/page
+https://another.example/path
+https://another.example/blog/post
+```
+
+**Reduction**: 8 input URLs → 6 sampled URLs (2 per domain)
+
+For CLI sampling, see the [CLI Reference](../usage/cli.md).
+

docs/source/api/settings.md

@@ -0,0 +1,32 @@
+# courlan.settings
+
+Configuration constants for URL filtering and content detection.
+
+```{automodule} courlan.settings
+:members:
+:undoc-members:
+:show-inheritance:
+```
+
+## Settings reference
+
+| Name | Type | Purpose |
+|------|------|---------|
+| `BLACKLIST` | `set[str]` | Domain fragments to exclude (social media, CDNs, e-commerce, etc.) |
+| `ALLOWED_PARAMS` | `set[str]` | Query parameters preserved during cleaning (content IDs, pagination) |
+| `LANG_PARAMS` | `set[str]` | Query parameter names used for language detection (e.g. `lang`, `language`) |
+| `TARGET_LANGS` | `dict[str, set[str]]` | ISO 639-1 codes mapped to accepted variants (e.g. `"de"` → `{"de", "deutsch", "ger"}`) |
+
+## Customizing Settings
+
+Settings are module-level objects loaded at import time. Patch them at runtime before any filtering calls:
+
+```python
+import courlan.settings as settings
+
+settings.BLACKLIST.add("myservice.com")
+settings.ALLOWED_PARAMS.add("story_id")
+settings.TARGET_LANGS["fr"].add("français")
+```
+
+For permanent changes, edit `courlan/settings.py` directly and reinstall in editable mode (`pip install -e .`).