diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 92001c11..0952ce2f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,6 +59,13 @@ jobs: run: | python -m pytest --cov=./ --cov-report=xml + - name: Test docs + # version matches .readthedocs.yml + if: ${{ matrix.python-version == '3.13' }} + run: | + uv pip install --system -e ".[docs]" + pytest docs/test_docs.py + # coverage - name: Upload coverage to Codecov if: ${{ matrix.python-version == '3.13' }} diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..dca58cd4 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,19 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.13" + +python: + install: + - method: pip + path: . + extra_requirements: + - docs + +sphinx: + configuration: docs/source/conf.py diff --git a/README.md b/README.md index 7b1223da..c0dfdfe1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Python package](https://img.shields.io/pypi/v/courlan.svg)](https://pypi.python.org/pypi/courlan) [![Python versions](https://img.shields.io/pypi/pyversions/courlan.svg)](https://pypi.python.org/pypi/courlan) [![Code Coverage](https://img.shields.io/codecov/c/github/adbar/courlan.svg)](https://codecov.io/gh/adbar/courlan) -[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +[![Documentation](https://readthedocs.org/projects/courlan/badge/?version=latest)](http://courlan.readthedocs.org/en/latest/) ## Why coURLan? diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..cfd15f64 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,11 @@ +SPHINXBUILD = sphinx-build +SOURCEDIR = docs/source +BUILDDIR = docs/_build + +.PHONY: html +html: + $(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)/html" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* diff --git a/docs/source/_static/.gitkeep b/docs/source/_static/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docs/source/api/clean.md b/docs/source/api/clean.md new file mode 100644 index 00000000..14433d4a --- /dev/null +++ b/docs/source/api/clean.md @@ -0,0 +1,24 @@ +# courlan.clean + +Core URL cleaning and normalization utilities. + +```{automodule} courlan.clean +:members: +:undoc-members: +:show-inheritance: +``` + +## Common usage + +```python +from courlan import clean_url, scrub_url, normalize_url, validate_url + +# Clean and normalize a URL (returns str or None if invalid) +url = clean_url('HTTPS://WWW.EXAMPLE.COM:443/path?utm_source=x') + +# Basic validation +is_valid, parsed = validate_url('https://example.com') + +# Normalization only +normalized = normalize_url('http://example.com/path?z=1&a=2#fragment') +``` diff --git a/docs/source/api/cli.md b/docs/source/api/cli.md new file mode 100644 index 00000000..3e323aab --- /dev/null +++ b/docs/source/api/cli.md @@ -0,0 +1,11 @@ +# courlan.cli + +Command-line interface implementation and argument parsing. + +This module contains the CLI entry point and internal helpers. Most users interact with this through the `courlan` command rather than importing it directly — see the [CLI Reference](../usage/cli.md) for full flag documentation. + +```{automodule} courlan.cli +:members: +:undoc-members: +:show-inheritance: +``` diff --git a/docs/source/api/core.md b/docs/source/api/core.md new file mode 100644 index 00000000..0afb2b23 --- /dev/null +++ b/docs/source/api/core.md @@ -0,0 +1,38 @@ +# courlan.core + +Core URL checking utilities. + +```{automodule} courlan.core +:members: +:undoc-members: +:show-inheritance: +``` + +## Common usage + +```python +from courlan import check_url, extract_links, filter_links + +# check_url returns (url, domain) or None if the URL is rejected +result = check_url('https://example.com/article') +if result: + url, domain = result + +# Strict mode and language filtering +result = check_url('https://example.com/article', strict=True, language='en') + +# Extract links from HTML (returns a set) +links = extract_links(html, 'https://example.com', external_bool=False) + +# Extract and prioritize links for crawling (returns links, priority_links) +links, priority_links = filter_links(html, 'https://example.com', lang='en') +``` + +## Filtering cost + +Options add overhead in this order, from cheapest to most expensive: + +1. **Basic** — `check_url(url)` +2. **Language filtering** — `check_url(url, language='en')` — minimal overhead +3. **Strict mode** — `check_url(url, strict=True)` — more conditions checked +4. **Redirect checks** — `check_url(url, with_redirects=True)` — network I/O; avoid on large datasets diff --git a/docs/source/api/filters.md b/docs/source/api/filters.md new file mode 100644 index 00000000..3a4ce580 --- /dev/null +++ b/docs/source/api/filters.md @@ -0,0 +1,32 @@ +# courlan.filters + +URL filtering heuristics for content validation and crawler optimization. + +```{automodule} courlan.filters +:members: +:undoc-members: +:show-inheritance: +``` + +## Common usage + +```python +from courlan import check_url, filter_links, lang_filter, is_valid_url + +# check_url returns (url, domain) or None if rejected +result = check_url('https://example.com/article', language='en', strict=True) +if result: + url, domain = result + +# Extract and filter links from HTML +html = 'LinkTag' +links, priority_links = filter_links(html, 'https://example.com', lang='en') + +# Test if a URL matches a target language heuristically +if lang_filter('https://example.com/en/article', language='en'): + print("Language matches") + +# Basic structural validity check (no network call) +if is_valid_url('https://example.com/path'): + print("Valid URL structure") +``` diff --git a/docs/source/api/index.md b/docs/source/api/index.md new file mode 100644 index 00000000..ebe3d539 --- /dev/null +++ b/docs/source/api/index.md @@ -0,0 +1,20 @@ +# API Reference + +This section is generated from the courlan package. Click a module to jump to its reference. + +```{toctree} +:maxdepth: 1 +:caption: Modules + +clean +cli +core +filters +meta +network +sampling +settings +urlstore +urlutils +``` + diff --git a/docs/source/api/meta.md b/docs/source/api/meta.md new file mode 100644 index 00000000..6dd052d8 --- /dev/null +++ b/docs/source/api/meta.md @@ -0,0 +1,66 @@ +# courlan.meta + +Cache management and meta-utilities. + +```{automodule} courlan.meta +:members: +:undoc-members: +:show-inheritance: +``` + +## Cache Management + +Courlan uses LRU (Least Recently Used) caches to speed up URL parsing and language detection. For long-running processes, you can clear these caches to reclaim memory. + +### clear_caches() + +**Purpose**: Reset all internal LRU caches. + +Use in long-running processes handling many distinct URLs, in memory-constrained environments, or between crawl phases. + +**What gets cleared**: urllib.parse results, language detection scores. + +**Example**: +```python +from courlan import check_url +from courlan.meta import clear_caches + +for i in range(10000): + result = check_url(f'https://example.com/page{i}') + if (i + 1) % 1000 == 0: + clear_caches() +``` + +--- + +## Usage in Batch Workflows + +```python +from courlan import UrlStore, check_url +from courlan.meta import clear_caches + +store = UrlStore(compressed=True) +store.add_urls(many_urls) + +# Process in batches +batch_size = 5000 +processed = 0 + +while store.unvisited_websites_number() > 0: + for domain in store.get_unvisited_domains(): + url = store.get_url(domain) + if url: + check_url(url, strict=True, language='en') + processed += 1 + + # Clear caches periodically + if processed % batch_size == 0: + clear_caches() + print(f"Processed {processed} URLs, caches cleared") +``` + +--- + +## See Also + +- [Web Crawling guide](../usage/crawling.md) — cache clearing in crawler workflows diff --git a/docs/source/api/network.md b/docs/source/api/network.md new file mode 100644 index 00000000..592ee1ed --- /dev/null +++ b/docs/source/api/network.md @@ -0,0 +1,19 @@ +# courlan.network + +Network helpers for redirect checking and HTTP operations. + +```{automodule} courlan.network +:members: +:undoc-members: +:show-inheritance: +``` + +## Common usage + +```python +# Redirect checking is typically used through check_url() function +from courlan import check_url + +# Check if URL redirects (makes HTTP HEAD request) +url, domain = check_url('https://example.com/old-page', with_redirects=True) +``` diff --git a/docs/source/api/sampling.md b/docs/source/api/sampling.md new file mode 100644 index 00000000..03e58e07 --- /dev/null +++ b/docs/source/api/sampling.md @@ -0,0 +1,74 @@ +# courlan.sampling + +Sampling utilities to produce per-host URL samples. + +```{automodule} courlan.sampling +:members: +:undoc-members: +:show-inheritance: +``` + +## Common usage + +```python +from courlan import sample_urls + +# Generate sample: up to 10 URLs per domain +urls = ['https://example.com/p1', 'https://example.com/p2', 'https://other.org/a'] +sample = sample_urls(urls, 10) + +# With exclusion filters +sample = sample_urls(urls, samplesize=5, exclude_min=2, exclude_max=100) +``` + +## Example: Sampling output + +**Input URLs** (8 total, 3 domains): + +``` +https://github.com/adbar/courlan +https://github.com/adbar/trafilatura +https://github.com/adbar/htmldate +https://example.com/some/page +https://example.com/another/page +https://example.com/third/page +https://another.example/path +https://another.example/blog/post +``` + +**Sample with `samplesize=2`** (2 URLs per domain): + +```python +from courlan import sample_urls + +urls = [ + 'https://github.com/adbar/courlan', + 'https://github.com/adbar/trafilatura', + 'https://github.com/adbar/htmldate', + 'https://example.com/some/page', + 'https://example.com/another/page', + 'https://example.com/third/page', + 'https://another.example/path', + 'https://another.example/blog/post', +] + +sample = sample_urls(urls, samplesize=2) +# Result: 6 URLs (2 per domain) +for url in sample: + print(url) +``` + +**Output**: +``` +https://github.com/adbar/courlan +https://github.com/adbar/trafilatura +https://example.com/some/page +https://example.com/another/page +https://another.example/path +https://another.example/blog/post +``` + +**Reduction**: 8 input URLs → 6 sampled URLs (2 per domain) + +For CLI sampling, see the [CLI Reference](../usage/cli.md). + diff --git a/docs/source/api/settings.md b/docs/source/api/settings.md new file mode 100644 index 00000000..8e463aa7 --- /dev/null +++ b/docs/source/api/settings.md @@ -0,0 +1,32 @@ +# courlan.settings + +Configuration constants for URL filtering and content detection. + +```{automodule} courlan.settings +:members: +:undoc-members: +:show-inheritance: +``` + +## Settings reference + +| Name | Type | Purpose | +|------|------|---------| +| `BLACKLIST` | `set[str]` | Domain fragments to exclude (social media, CDNs, e-commerce, etc.) | +| `ALLOWED_PARAMS` | `set[str]` | Query parameters preserved during cleaning (content IDs, pagination) | +| `LANG_PARAMS` | `set[str]` | Query parameter names used for language detection (e.g. `lang`, `language`) | +| `TARGET_LANGS` | `dict[str, set[str]]` | ISO 639-1 codes mapped to accepted variants (e.g. `"de"` → `{"de", "deutsch", "ger"}`) | + +## Customizing Settings + +Settings are module-level objects loaded at import time. Patch them at runtime before any filtering calls: + +```python +import courlan.settings as settings + +settings.BLACKLIST.add("myservice.com") +settings.ALLOWED_PARAMS.add("story_id") +settings.TARGET_LANGS["fr"].add("français") +``` + +For permanent changes, edit `courlan/settings.py` directly and reinstall in editable mode (`pip install -e .`). diff --git a/docs/source/api/urlstore.md b/docs/source/api/urlstore.md new file mode 100644 index 00000000..98bbf829 --- /dev/null +++ b/docs/source/api/urlstore.md @@ -0,0 +1,111 @@ +# courlan.urlstore + +Domain-classified URL storage for web crawling workflows. + +```{automodule} courlan.urlstore +:members: +:undoc-members: +:show-inheritance: +``` + +For crawler-oriented usage (crawl loops, scheduling, robots.txt, HTML link extraction), see the [Web Crawling guide](../usage/crawling.md). + +## Examples + +### Basic URL tracking + +```python +from courlan import UrlStore + +store = UrlStore() +store.add_urls([ + 'https://example.com/page1', + 'https://example.com/page2', + 'https://example.org/article', +]) + +while store.unvisited_websites_number() > 0: + for domain in store.get_unvisited_domains(): + url = store.get_url(domain) # marks URL as visited + if url: + print(f"Processing: {url}") + store.add_urls(['https://example.com/page3']) +``` + +### Persistent store (save/load) + +```python +from courlan import UrlStore, load_store + +# Build store over time +store = UrlStore() +store.add_urls(['https://example.com/1', 'https://example.com/2']) +store.get_url('https://example.com') # mark as visited + +# Save to disk +store.write('my_urls.db') + +# Later: load from disk (different session) +store = load_store('my_urls.db') + +# Continue where you left off +print(f"Total URLs: {store.total_url_number()}") +print(f"Unvisited domains: {store.get_unvisited_domains()}") +``` + +### Statistics and reporting + +```python +from courlan import UrlStore + +store = UrlStore() +store.add_urls([ + 'https://a.com/1', 'https://a.com/2', 'https://a.com/3', + 'https://b.org/x', 'https://b.org/y', + 'https://c.net/article', +]) + +# Mark some as visited +store.get_url('https://a.com') +store.get_url('https://a.com') + +# Generate statistics +print(f"Total URLs: {store.total_url_number()}") +print(f"Known domains: {store.get_known_domains()}") +print(f"Unvisited domains: {store.get_unvisited_domains()}") + +# Per-domain stats +for domain in store.get_known_domains(): + all_urls = store.find_known_urls(domain) + unvisited = store.find_unvisited_urls(domain) + print(f"{domain}: {len(all_urls)} total, {len(unvisited)} unvisited") +``` + +### Filtering and deduplication + +```python +from courlan import UrlStore + +store = UrlStore() +store.add_urls(['https://example.com/page', 'https://example.org/post']) + +# Check if URL is already known +if store.is_known('https://example.com/page'): + print("Already in store") + +# Filter unknown URLs +new_urls = ['https://example.com/page', 'https://example.com/new'] +unknown = store.filter_unknown_urls(new_urls) +print(f"Unknown URLs: {unknown}") + +# Filter unvisited URLs +unvisited = store.filter_unvisited_urls(new_urls) +``` + +## Performance tips + +- **For large crawls**: Use `compressed=True` to reduce memory +- **Storage**: Save the store periodically with `write(filename)` +- **Scheduling**: Use `establish_download_schedule()` to respect crawl delays +- **Languages**: Set language filter at init to filter links automatically: `UrlStore(language='en')` + diff --git a/docs/source/api/urlutils.md b/docs/source/api/urlutils.md new file mode 100644 index 00000000..95c72ee7 --- /dev/null +++ b/docs/source/api/urlutils.md @@ -0,0 +1,34 @@ +# courlan.urlutils + +URL parsing, decomposition, and relative URL resolution utilities. + +```{automodule} courlan.urlutils +:members: +:undoc-members: +:show-inheritance: +``` + +## Common usage + +```python +from courlan import extract_domain, get_base_url, get_host_and_path, fix_relative_urls +from courlan import get_hostinfo, filter_urls + +# Extract domain from URL +domain = extract_domain('https://www.example.com/path', fast=True) + +# Get base URL (scheme + netloc) +base = get_base_url('https://example.com/path/page?q=1') + +# Decompose URL into host and path +host, path = get_host_and_path('https://example.com/articles/post') + +# Convenience: domain name + base URL in one call +domainname, base_url = get_hostinfo('https://www.example.com/path') + +# Resolve relative URLs +absolute = fix_relative_urls('https://example.com', 'articles/post.html') + +# Filter a list of URLs by substring pattern (None = deduplicate only) +subset = filter_urls(link_list, urlfilter='example.com') +``` diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..b35434f4 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,45 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../../")) + +from courlan import __author__, __version__ + +project = "courlan" +author = __author__ +release = __version__ + +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", + "sphinx_copybutton", +] + +napoleon_google_docstring = True +napoleon_numpy_docstring = False +autodoc_typehints = "description" + +autodoc_default_options = { + "members": True, + "undoc-members": True, + "show-inheritance": True, +} + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +myst_enable_extensions = [ + "deflist", + "colon_fence", +] + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] +html_title = project + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), +} diff --git a/docs/source/getting-started.md b/docs/source/getting-started.md new file mode 100644 index 00000000..fd8e5b1b --- /dev/null +++ b/docs/source/getting-started.md @@ -0,0 +1,97 @@ +# Getting Started + +This single guide covers both installation and a minimal, runnable Quickstart. + +## Prerequisites +- Python 3.10+ + + +## Install + +Install the latest release from PyPI, e.g. with pip or uv: + +```bash +pip install courlan +``` + +Or install from source for development: + +```bash +git clone https://github.com/adbar/courlan.git +cd courlan +pip install -e . +``` + +## Minimal CLI Quickstart (hands-on) + +1) Create a simple input file (one URL per line): + +```bash +cat > urls.txt <<'EOF' +https://github.com/adbar/courlan +https://example.com/some/page +https://another.example/path +EOF +``` + +2) Run a full processing pass (filters + optional redirect checks). `-i/--inputfile` and `-o/--outputfile` are required: + +```bash +courlan -i urls.txt -o cleaned.txt +# or +courlan --inputfile urls.txt --outputfile cleaned.txt +``` + +Result: cleaned.txt contains accepted URLs (one per line). Exit code 0 indicates success. + + +## Troubleshooting & tips +- If the command fails, run `courlan --help` to inspect flags and check your input file encoding. +- For development, prefer `pip install -e .` so local changes take effect immediately. + +## End-to-end example + +Create an input file, filter it, and inspect the results: + +```bash +cat > urls.txt <<'EOF' +https://www.example.com/page1 +https://www.example.com/page2?utm_source=twitter +https://login.example.com/signin +https://cdn.example.com/image.jpg +https://example.org/valid-article +EOF + +courlan -i urls.txt -o cleaned.txt -d discarded.txt --strict -v +``` + +`cleaned.txt` — accepted URLs; `discarded.txt` — rejected ones (trackers, login pages, media files, etc.). + +Inspect from Python: + +```python +from courlan import check_url, UrlStore + +# check_url returns (url, domain) or None if rejected +result = check_url('https://example.org/valid-article') +if result: + url, domain = result + print(f"Accepted: {url} ({domain})") + +store = UrlStore() +with open('cleaned.txt') as f: + store.add_urls([line.strip() for line in f]) + +for domain in store.get_known_domains(): + print(f"{domain}: {len(store.find_known_urls(domain))} URL(s)") +``` + +Sample by domain for large lists: + +```bash +courlan -i urls.txt -o sample.txt --sample 5 --exclude-min 2 +``` + +## Where to go next +- **CLI Reference**: all flags and examples +- **API Reference**: programmatic integration diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 00000000..a80875db --- /dev/null +++ b/docs/source/index.md @@ -0,0 +1,14 @@ +# courlan + +courlan cleans, filters, normalizes, and samples URLs. It is designed as a building block for web crawlers and scrapers: steer clear of low-value pages, identify content by language, and deduplicate URL collections at scale. + +```{toctree} +:maxdepth: 2 +:caption: Contents + +getting-started +usage/cli +usage/crawling +api/index + +``` diff --git a/docs/source/usage/cli.md b/docs/source/usage/cli.md new file mode 100644 index 00000000..8d8cef64 --- /dev/null +++ b/docs/source/usage/cli.md @@ -0,0 +1,131 @@ +# CLI Reference + +The courlan command-line utility is installed as the `courlan` entry point. + +```bash +courlan -i INPUTFILE -o OUTPUTFILE [options] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `-i, --inputfile` | Input file — one URL per line (required) | +| `-o, --outputfile` | Output file (required) | +| `-d, --discardedfile` | Write rejected URLs to this file | +| `-v, --verbose` | Enable debug logging | +| `-p, --parallel` | Worker processes for batch mode (default: 1) | +| `--strict` | Enable more restrictive filtering | +| `-l, --language` | Keep only URLs matching this ISO 639-1 code (e.g. `en`, `de`) | +| `-r, --redirects` | Check HTTP redirects (slow — see below) | +| `--sample N` | Sample N URLs per domain instead of full processing | +| `--exclude-min N` | Skip domains with fewer than N URLs (sampling only) | +| `--exclude-max N` | Skip domains with more than N URLs (sampling only) | + +## Behavior + +- **Batch mode** (default): processes all URLs, writes accepted URLs to `--outputfile` and rejected ones to `--discardedfile` if specified. Parallelism controlled by `-p`. +- **Sampling mode** (`--sample`): samples N URLs per domain; `-p` is ignored. + +## Complete CLI examples + +### Example 1: Basic filtering with output capture + +**Input file** (`urls.txt`): +``` +https://www.example.com/page1 +https://www.example.com/page2 +https://example.com/archive +https://cdn.example.com/image.jpg +https://example.org/article +``` + +**Command**: +```bash +courlan -i urls.txt -o cleaned.txt -d discarded.txt +``` + +**Output files**: + +`cleaned.txt` (accepted URLs): +``` +https://www.example.com/page1 +https://www.example.com/page2 +https://example.org/article +``` + +`discarded.txt` (rejected URLs): +``` +https://example.com/archive +https://cdn.example.com/image.jpg +``` + +### Example 2: Strict filtering with language detection + +**Command**: +```bash +courlan -i urls.txt -o cleaned.txt -d discarded.txt --strict -l en +``` + +More restrictive filtering applied; only English URLs kept. + +### Example 3: Parallel processing with verbose output + +**Command** (4 worker processes, debug logging): +```bash +courlan -i urls.txt -o cleaned.txt -p 4 -v +``` + +Outputs debug information about each URL processing step. + +### Example 4: Sampling by domain + +**Input file** (`large_urls.txt`): +``` +https://github.com/adbar/courlan +https://github.com/adbar/trafilatura +https://github.com/adbar/htmldate +https://example.com/page1 +https://example.com/page2 +https://example.com/page3 +https://another.org/article +``` + +**Command** (2 URLs per domain, exclude domains with <2 URLs): +```bash +courlan -i large_urls.txt -o sample.txt --sample 2 --exclude-min 2 +``` + +**Output** (`sample.txt`): +``` +https://github.com/adbar/courlan +https://github.com/adbar/trafilatura +https://example.com/page1 +https://example.com/page2 +``` + +Result: 4 URLs selected (2 per domain that meets the exclusion criteria). + +## Large inputs + +For very large files (>1M URLs), split into chunks to limit memory usage: + +```bash +split -l 100000 urls.txt urls_chunk_ +for chunk in urls_chunk_*; do + courlan -i "$chunk" -o "out_$chunk" -p 4 +done +``` + +## Redirect checking (`-r`) + +Redirect checks require an HTTP HEAD request per URL and can be slow. + +**Use when:** +- You need to resolve redirect chains +- The dataset is small (<10k URLs) + +**Avoid when:** +- Doing initial bulk filtering +- Processing large URL lists (>100k) +- Network latency is a concern diff --git a/docs/source/usage/crawling.md b/docs/source/usage/crawling.md new file mode 100644 index 00000000..01477d92 --- /dev/null +++ b/docs/source/usage/crawling.md @@ -0,0 +1,230 @@ +# Web Crawling with Courlan + +Guide to building web crawlers with courlan: frontier management, crawl delays, link extraction, and persistence. + +## UrlStore for Crawler State + +The `UrlStore` class manages the crawl frontier: tracking visited/unvisited URLs per domain and handling robots.txt rules. + +### Basic Crawler Loop + +```python +from courlan import UrlStore + +store = UrlStore(language='en', strict=True) +store.add_urls([ + 'https://example.com/page1', + 'https://example.com/page2', + 'https://other.org/article', +]) + +while store.unvisited_websites_number() > 0: + for domain in store.get_unvisited_domains(): + url = store.get_url(domain) # marks as visited + if not url: + continue + print(f"Visiting: {url}") + # response = requests.get(url, timeout=10) + # store.add_urls(extract_links(response.text, url)) +``` + +### Key UrlStore Methods + +| Method | Purpose | +|--------|---------| +| `add_urls(urls)` | Add URLs to frontier | +| `get_url(domain)` | Retrieve next URL and mark as visited | +| `get_unvisited_domains()` | Domains with unvisited URLs | +| `unvisited_websites_number()` | Count of domains with remaining URLs | +| `establish_download_schedule(max_urls, time_limit)` | Batch URLs with per-domain delays | +| `download_threshold_reached(threshold)` | Check if time limit exceeded | +| `find_unvisited_urls(domain)` | List unvisited URLs for a domain | +| `is_exhausted_domain(domain)` | Check if domain has no more URLs | +| `write(filename)` | Save state to disk | + +--- + +## Crawl Delays + +Use `get_crawl_delay()` to read the delay from stored robots.txt rules, and `store_rules()` / `get_rules()` to persist them. + +```python +from courlan import UrlStore +from urllib.robotparser import RobotFileParser +import time + +store = UrlStore() +domain = 'https://example.com' + +# Store robots.txt rules after fetching +rules = RobotFileParser(f'{domain}/robots.txt') +rules.read() +store.store_rules(domain, rules) + +# Apply delay between requests +delay = store.get_crawl_delay(domain, default=5) +time.sleep(delay) +url = store.get_url(domain) +``` + +### Scheduled Download Strategy + +For large crawls, `establish_download_schedule()` batches URLs with appropriate per-domain delays: + +```python +from courlan import UrlStore +import time + +store = UrlStore() +store.add_urls([ + 'https://a.com/1', 'https://a.com/2', + 'https://b.org/x', 'https://b.org/y', +]) + +schedule = store.establish_download_schedule(max_urls=100, time_limit=10) + +for delay, url in schedule: + time.sleep(delay) + print(f"Fetching: {url}") + # response = requests.get(url) + if store.download_threshold_reached(threshold=60): + break +``` + +--- + +## Crawler Frontier Management + +### Scope Detection + +```python +from courlan import is_external + +if not is_external(found_url, 'https://example.com', ignore_suffix=False): + store.add_urls([found_url]) +``` + +### Navigation Page Detection + +```python +from courlan import is_navigation_page + +for url in candidate_urls: + if not is_navigation_page(url): + store.add_urls([url]) # content page, high priority +``` + +### Crawlability Detection + +```python +from courlan import is_not_crawlable + +for url in candidate_urls: + if not is_not_crawlable(url): + store.add_urls([url]) +``` + +--- + +## Extracting Links from HTML + +```python +from courlan import extract_links + +links = extract_links( + html, + base_url, + external_bool=False, + language='en', + strict=True, +) +store.add_urls(links) +``` + +`extract_links` also accepts `no_filter`, `redirects`, and `with_nav` — see the API reference for details. + +--- + +## Persistence and Resume + +```python +from courlan import UrlStore, load_store + +store = UrlStore() +store.add_urls(['https://example.com/page1', 'https://example.com/page2']) +store.get_url('https://example.com') + +store.write('crawler_state.db') + +# Later session: +store = load_store('crawler_state.db') +print(f"Unvisited domains: {store.get_unvisited_domains()}") +``` + +--- + +## Best Practices + +| Practice | Reason | +|----------|--------| +| Respect robots.txt | Legal/ethical requirement | +| Set crawl delays | Avoid overloading servers | +| Identify User-Agent | Tell servers who you are | +| Save crawler state | Resume after interruptions | +| Skip navigation pages | Focus on content | +| Validate URLs | Avoid malformed requests | +| Handle errors gracefully | Don't crash on bad pages | +| Limit crawl scope | Stay on target domain(s) | + +--- + +## Complete Example + +```python +from courlan import UrlStore, extract_links, is_not_crawlable +import time + +store = UrlStore(language='en', strict=True) +store.add_urls(['https://example.com']) +pages_crawled = 0 + +while store.unvisited_websites_number() > 0 and pages_crawled < 100: + for domain in store.get_unvisited_domains(): + url = store.get_url(domain) + if not url or is_not_crawlable(url): + continue + try: + # response = requests.get(url, timeout=10) + # links = extract_links(response.text, url, external_bool=False) + # store.add_urls(links) + pages_crawled += 1 + time.sleep(2) + except Exception as e: + print(f"Error: {url} - {e}") + +store.write('crawler_state.db') +``` + +--- + +## Troubleshooting + +**URL not added to store** — `UrlStore.add_urls()` silently drops invalid URLs. Validate first: + +```python +from courlan import check_url + +if check_url(url, strict=True) is None: + print("URL failed validation") +``` + +**Memory growing during a long crawl** — use `compressed=True` and periodically clear caches: + +```python +from courlan import UrlStore +from courlan.meta import clear_caches + +store = UrlStore(compressed=True) +# ... process URLs ... +clear_caches() +``` diff --git a/docs/test_docs.py b/docs/test_docs.py new file mode 100644 index 00000000..e3eaae3d --- /dev/null +++ b/docs/test_docs.py @@ -0,0 +1,27 @@ +import shutil +import subprocess +from pathlib import Path + +import pytest + +DOCS_SOURCE = Path(__file__).parent / "source" +DOCS_BUILD = Path(__file__).parent / "_build" +DOCS_HTML = DOCS_BUILD / "html" + + +@pytest.fixture(autouse=True, scope="module") +def clean_build(): + if DOCS_BUILD.exists(): + shutil.rmtree(DOCS_BUILD) + yield + + +def test_sphinx_build_succeeds(): + cmd = ["sphinx-build", "-W", "-b", "html", str(DOCS_SOURCE), str(DOCS_HTML)] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) + if result.returncode != 0: + pytest.fail( + f"Sphinx build failed (exit {result.returncode})\n" + f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + assert (DOCS_HTML / "index.html").exists() diff --git a/pyproject.toml b/pyproject.toml index b08676fe..92a789bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,12 @@ dev = [ "pytest-cov==7.1.0", "pytest-httpserver==1.1.5", ] +docs = [ + "sphinx>=6.2", + "myst-parser>=0.19.0", + "sphinx-rtd-theme>=1.2.0", + "sphinx-copybutton>=0.5.0", +] [tool.pytest.ini_options] testpaths = "tests/*test*.py"