diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 734618daa..d8d51047b 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -20,9 +20,19 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Check out audience statistics + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: AA-Turner/plausible-stats + path: plausible-stats + sparse-checkout: stats/docs.python.org_*/*.visitors.json + sparse-checkout-cone-mode: false + persist-credentials: false - run: sudo apt-get install -y gettext - run: pip install -r requirements.txt - run: uv run generate.py # generates index.html and index.json + env: + PLAUSIBLE_STATS_DIR: plausible-stats/stats - name: Deploy 🚀 if: github.event_name != 'pull_request' uses: JamesIves/github-pages-deploy-action@d92aa235d04922e8f08b40ce78cc5442fcfbfa2f # v4.8.0 diff --git a/audience.py b/audience.py new file mode 100644 index 000000000..f03e1a549 --- /dev/null +++ b/audience.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import json +import logging +from collections.abc import Iterable +from pathlib import Path + + +def get_audience( + stats_dir: Path, + language_codes: Iterable[str], + snapshots: int = 30, + *, + required: bool = False, +) -> dict[str, int]: + """Return a rolling audience score for each language. + + Plausible's public exports contain one visitor count per hour, so the score + is the sum of those counts across the most recent available snapshots. + """ + if snapshots < 1: + raise ValueError('snapshots must be at least 1') + + audience = dict.fromkeys(language_codes, 0) + if not stats_dir.is_dir(): + message = f'Plausible statistics directory not found: {stats_dir}' + if required: + raise FileNotFoundError(message) + logging.warning(message) + return audience + + snapshot_dirs = sorted( + (path for path in stats_dir.glob('docs.python.org_*') if path.is_dir()), + key=lambda path: path.name, + reverse=True, + ) + if not snapshot_dirs: + message = f'No Plausible statistics snapshots found in {stats_dir}' + if required: + raise FileNotFoundError(message) + logging.warning(message) + return audience + + snapshots_read = dict.fromkeys(audience, 0) + files_read = 0 + for snapshot_dir in snapshot_dirs: + for language_code in audience: + if snapshots_read[language_code] >= snapshots: + continue + visitors_file = snapshot_dir / ( + f'{snapshot_dir.name}.prefix-{language_code}.visitors.json' + ) + if not visitors_file.exists(): + continue + + rows = json.loads(visitors_file.read_text()) + audience[language_code] += sum(int(row['visitors']) for row in rows) + snapshots_read[language_code] += 1 + files_read += 1 + + if required and not files_read: + raise FileNotFoundError(f'No Plausible visitor statistics found in {stats_dir}') + + return audience diff --git a/generate.py b/generate.py index a6df9c6f3..86f4fb8ca 100644 --- a/generate.py +++ b/generate.py @@ -4,6 +4,7 @@ import concurrent.futures import itertools import logging +import os import subprocess from collections.abc import Iterator from dataclasses import dataclass, asdict @@ -16,10 +17,14 @@ import translated_names import contribute +from audience import get_audience from completion import branches_from_peps, get_completion from repositories import Language, get_languages_and_repos generation_time = datetime.now(timezone.utc) +default_plausible_stats_dir = ( + Path(__file__).resolve().parent.parent / 'plausible-stats' / 'stats' +) def get_completion_progress() -> Iterator[LanguageProjectData]: @@ -49,13 +54,21 @@ def get_completion_progress() -> Iterator[LanguageProjectData]: language: translated_name for language, translated_name in translated_names.get_languages(PoolManager()) } + languages_and_repos = list(get_languages_and_repos(devguide_dir)) + plausible_stats_dir = os.environ.get('PLAUSIBLE_STATS_DIR') + audience = get_audience( + Path(plausible_stats_dir or default_plausible_stats_dir), + (language.code for language, _ in languages_and_repos), + required=plausible_stats_dir is not None, + ) with concurrent.futures.ThreadPoolExecutor() as executor: return executor.map( get_project_data, - *zip(*get_languages_and_repos(devguide_dir)), + *zip(*languages_and_repos), itertools.repeat(languages_built), itertools.repeat(clones_dir), + itertools.repeat(audience), ) @@ -64,6 +77,7 @@ def get_project_data( repo: str | None, languages_built: dict[str, str], clones_dir: str, + audience: dict[str, int], ) -> LanguageProjectData: built = language.code in languages_built if repo: @@ -88,6 +102,7 @@ def get_project_data( or translated_names.babel_autonym(language.code) or '', contribution_link=contribute.get_contrib_link(language.code, repo), + audience=audience[language.code], ) @@ -103,6 +118,7 @@ class LanguageProjectData: built: bool translated_name: str contribution_link: str | None + audience: int if __name__ == '__main__': diff --git a/templates/index.html.jinja b/templates/index.html.jinja index 9b5897455..e02405cb8 100644 --- a/templates/index.html.jinja +++ b/templates/index.html.jinja @@ -1,9 +1,32 @@ {% extends "base.html.jinja" %} {% block main %} -
+
+
+
+
+ + +
+
+
+
+
- {% for project in completion_progress | sort(attribute='core_completion,completion') | reverse %} -
+ {% for project in completion_progress | sort(attribute='completion') | reverse %} +

@@ -63,26 +86,69 @@ }); } + function sortLanguages(sortBy) { + const languageContainer = document.getElementById('languageContainer'); + if (!languageContainer) { + return; + } + const row = languageContainer.querySelector('.row'); + if (!row) { + return; + } + + const sortFields = { + 'completion': 'completion', + 'core-completion': 'coreCompletion', + 'progress': 'progress', + 'audience': 'audience', + }; + const cards = Array.from(row.children); + cards.sort((a, b) => { + const aPinPriority = a.dataset.pinPriority; + const bPinPriority = b.dataset.pinPriority; + if (aPinPriority !== undefined || bPinPriority !== undefined) { + if (aPinPriority === undefined) return 1; + if (bPinPriority === undefined) return -1; + return Number(aPinPriority) - Number(bPinPriority); + } + + const nameComparison = a.dataset.name.localeCompare(b.dataset.name); + if (sortBy === 'name') { + return nameComparison; + } + const sortField = sortFields[sortBy] || sortFields.completion; + return Number(b.dataset[sortField]) - Number(a.dataset[sortField]) || nameComparison; + }); + + row.innerHTML = ''; + cards.forEach(card => row.appendChild(card)); + } + updateProgressBarVisibility(); + const sortDropdown = document.getElementById('sortDropdown'); + if (sortDropdown) { + sortDropdown.addEventListener('change', function() { + sortLanguages(this.value); + }); + sortLanguages(sortDropdown.value); + } + window.addEventListener('resize', updateProgressBarVisibility); (function () { const userLangs = Array.from(navigator.languages || []).map(lang => lang.toLowerCase()); - const row = document.querySelector('.row'); + const row = document.querySelector('#languageContainer > .row'); if (!row || !userLangs.length) return; - // Capture the original column order (completion-based sort from server) before any changes. - const originalCols = Array.from(row.children); - // Find the first matching card column for each user language preference. const userLangToCol = new Map(); for (const lang of userLangs) { const langBase = lang.split('-')[0]; const card = row.querySelector(`[id="${lang}"]`) || row.querySelector(`[id="${langBase}"]`); if (card) { - const col = card.closest('.col-12'); + const col = card.parentElement; if (col && col.parentElement === row) { userLangToCol.set(lang, col); } @@ -106,7 +172,10 @@ // localStorage helpers – store the set of explicitly unpinned card IDs. const LS_KEY = 'dashboard-unpinned'; function getUnpinned() { - try { return new Set(JSON.parse(localStorage.getItem(LS_KEY) || '[]')); } + try { + const value = JSON.parse(localStorage.getItem(LS_KEY) || '[]'); + return Array.isArray(value) ? new Set(value) : new Set(); + } catch { return new Set(); } } function saveUnpinned(set) { @@ -128,8 +197,10 @@ for (const { cardId, col } of uniqueUserCols) { const card = col.querySelector('.card'); const btn = document.createElement('button'); + const languageName = card.querySelector('.card-title a')?.textContent?.trim() || cardId; btn.className = 'pin-btn'; btn.dataset.cardId = cardId; + btn.dataset.languageName = languageName; btn.innerHTML = `${PIN_FILL}${PIN_OUTLINE}`; card.appendChild(btn); @@ -141,34 +212,36 @@ unpinned.add(cardId); } saveUnpinned(unpinned); - reorder(); + reorder(unpinned); }); } - function reorder() { - const unpinned = getUnpinned(); + function reorder(unpinned = getUnpinned()) { // Pinned user-language columns in navigator.languages priority order. const pinnedCols = uniqueUserCols .filter(({ cardId }) => !unpinned.has(cardId)) .map(({ col }) => col); - - // Remaining columns in original server-side sort order. - const pinnedSet = new Set(pinnedCols); - const restCols = originalCols.filter(col => !pinnedSet.has(col)); - - [...pinnedCols, ...restCols].forEach(col => row.appendChild(col)); + for (const col of row.children) { + delete col.dataset.pinPriority; + } + pinnedCols.forEach((col, index) => { + col.dataset.pinPriority = index; + }); // Sync button appearance and accessible label. for (const { cardId, col } of uniqueUserCols) { const btn = col.querySelector('.pin-btn'); if (btn) { const isPinned = !unpinned.has(cardId); + const languageName = btn.dataset.languageName || cardId; btn.classList.toggle('unpinned', !isPinned); - btn.title = isPinned ? 'Unpin' : 'Pin'; - btn.setAttribute('aria-label', isPinned ? 'Unpin this language card' : 'Pin this language card'); + btn.title = isPinned ? `Unpin ${languageName}` : `Pin ${languageName}`; + btn.setAttribute('aria-label', isPinned ? `Unpin ${languageName} language card` : `Pin ${languageName} language card`); } } + + sortLanguages(sortDropdown?.value || 'completion'); } reorder(); diff --git a/tests/test_audience.py b/tests/test_audience.py new file mode 100644 index 000000000..d68561b98 --- /dev/null +++ b/tests/test_audience.py @@ -0,0 +1,74 @@ +import json +import tempfile +import unittest +from pathlib import Path + +import support + +with support.import_scripts(): + from audience import get_audience + + +class TestAudience(unittest.TestCase): + def setUp(self): + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.stats_dir = Path(self.temporary_directory.name) + + def write_visitors(self, date, language_code, visitors): + snapshot_name = f'docs.python.org_{date}' + snapshot_dir = self.stats_dir / snapshot_name + snapshot_dir.mkdir(exist_ok=True) + ( + snapshot_dir / f'{snapshot_name}.prefix-{language_code}.visitors.json' + ).write_text(json.dumps([{'visitors': value} for value in visitors])) + + def test_sums_hourly_visitors_from_most_recent_snapshots(self): + self.write_visitors('2026-08-20', 'pl', ['2', '3']) + self.write_visitors('2026-08-21', 'pl', ['5', '7']) + self.write_visitors('2026-08-22', 'pl', ['11', '13']) + + audience = get_audience(self.stats_dir, ['pl'], snapshots=2) + + self.assertEqual(audience, {'pl': 36}) + + def test_supports_hyphenated_codes_and_missing_languages(self): + self.write_visitors('2026-08-22', 'pt-br', ['17']) + + audience = get_audience(self.stats_dir, ['pt-br', 'de']) + + self.assertEqual(audience, {'pt-br': 17, 'de': 0}) + + def test_uses_older_snapshot_when_newer_one_is_missing(self): + self.write_visitors('2026-08-20', 'pl', ['2']) + self.write_visitors('2026-08-21', 'es', ['100']) + self.write_visitors('2026-08-22', 'pl', ['5']) + + audience = get_audience(self.stats_dir, ['pl'], snapshots=2) + + self.assertEqual(audience, {'pl': 7}) + + def test_missing_stats_directory_returns_zero(self): + audience = get_audience(self.stats_dir / 'missing', ['pl']) + + self.assertEqual(audience, {'pl': 0}) + + def test_malformed_schema_is_not_ignored(self): + self.write_visitors('2026-08-22', 'pl', [None]) + + with self.assertRaises(TypeError): + get_audience(self.stats_dir, ['pl']) + + def test_requires_at_least_one_snapshot(self): + with self.assertRaises(ValueError): + get_audience(self.stats_dir, ['pl'], snapshots=0) + + def test_required_stats_must_contain_visitor_files(self): + (self.stats_dir / 'docs.python.org_2026-08-22').mkdir() + + with self.assertRaises(FileNotFoundError): + get_audience(self.stats_dir, ['pl'], required=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_index.py b/tests/test_index.py index a1dbdcddb..446568a3d 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -23,12 +23,20 @@ def test_renders(self): built=True, translated_name='Polish', contribution_link='https://example.com', + audience=123, ) - env.get_template('index.html.jinja').render( + index = env.get_template('index.html.jinja').render( completion_progress=[language_project_data], generation_time=datetime.now(), duration=100, ) + self.assertIn('', index) + self.assertIn('', index) + self.assertIn('', index) + self.assertIn('data-core-completion="100"', index) + self.assertIn('data-progress="2"', index) + self.assertIn('data-audience="123"', index) + self.assertIn("document.querySelector('#languageContainer > .row')", index) if __name__ == '__main__':