diff --git a/src/api/routers/community.py b/src/api/routers/community.py index 0fc6426..37040b1 100644 --- a/src/api/routers/community.py +++ b/src/api/routers/community.py @@ -18,7 +18,7 @@ from pathlib import Path from typing import Annotated, Any, Literal -from fastapi import APIRouter, Header, HTTPException, Query, Request +from fastapi import APIRouter, Header, HTTPException, Query, Request, Response from fastapi.responses import FileResponse, StreamingResponse from langchain_core.messages import AIMessage, HumanMessage from langchain_core.messages.utils import count_tokens_approximately @@ -34,6 +34,7 @@ from src.assistants.registry import AssistantInfo from src.core.config.community import WidgetConfig from src.core.services.litellm_llm import create_openrouter_llm +from src.knowledge.search import FAQResult, get_citation_stats, list_faq_entries from src.metrics.cost import COST_BLOCK_THRESHOLD, COST_WARN_THRESHOLD, MODEL_PRICING, estimate_cost from src.metrics.db import ( RequestLogEntry, @@ -205,6 +206,75 @@ class CommunityConfigResponse(BaseModel): status: str = Field(..., description="Health status: healthy, degraded, or error") +class FAQEntryResponse(BaseModel): + """A single FAQ entry exposed via the public feed.""" + + question: str = Field(..., description="Synthesized question") + answer: str = Field(..., description="Synthesized answer") + tags: list[str] = Field(default_factory=list, description="Keyword tags") + category: str = Field(..., description="Entry category (how-to, troubleshooting, etc.)") + quality_score: float = Field(..., description="LLM quality score (0.0-1.0)") + message_count: int = Field(..., description="Number of source messages in the thread") + first_message_date: str = Field(..., description="Date of the first message in the thread") + thread_url: str = Field(..., description="URL of the source discussion thread") + + +class FAQFeedResponse(BaseModel): + """Paginated public FAQ feed for a community.""" + + community_id: str = Field(..., description="Community identifier") + total: int = Field(..., description="Total entries matching the filters") + limit: int = Field(..., description="Page size used for this response") + offset: int = Field(..., description="Offset used for this response") + entries: list[FAQEntryResponse] = Field(default_factory=list, description="FAQ entries") + + +class CitationsFeedResponse(BaseModel): + """Public citation dashboard data for a community's canonical papers.""" + + community_id: str = Field(..., description="Community identifier") + total: int = Field(..., description="Total citing papers with a recorded canonical link") + per_year: dict[str, int] = Field( + default_factory=dict, description="Citing-paper count per year across all papers" + ) + by_paper: dict[str, dict[str, int]] = Field( + default_factory=dict, + description="Stacked breakdown: canonical DOI -> year -> citing-paper count", + ) + canonical_dois: list[str] = Field( + default_factory=list, description="Canonical DOIs tracked for this community" + ) + + +# Matches bare email addresses so they can be stripped from the public feed. +_EMAIL_PATTERN = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") + + +def _redact_emails(text: str) -> str: + """Replace any email address in ``text`` with a redaction marker. + + The FAQ feed is derived from public mailing-list content. The summarizer + strips most personal data, but a handful of entries still embed addresses + (mostly vendor support lines). A public JSON feed should not emit raw + addresses, so they are redacted at serialization time. + """ + return _EMAIL_PATTERN.sub("[email redacted]", text) + + +def _faq_result_to_response(entry: FAQResult) -> FAQEntryResponse: + """Convert a knowledge-layer FAQResult into a public response model.""" + return FAQEntryResponse( + question=_redact_emails(entry.question), + answer=_redact_emails(entry.answer), + tags=[_redact_emails(tag) for tag in entry.tags], + category=entry.category, + quality_score=entry.quality_score, + message_count=entry.message_count, + first_message_date=entry.first_message_date, + thread_url=entry.thread_url, + ) + + # --------------------------------------------------------------------------- # Session Management (In-Memory, per-community isolation) # --------------------------------------------------------------------------- @@ -1502,6 +1572,116 @@ async def community_usage_public( detail="Metrics database is temporarily unavailable.", ) + @router.get("/faq", response_model=FAQFeedResponse) + async def community_faq( + response: Response, + q: str | None = Query( + default=None, + description="Optional full-text search phrase. If omitted, browses all entries.", + max_length=200, + ), + category: str | None = Query( + default=None, + description="Filter by category (how-to, troubleshooting, reference, etc.)", + max_length=50, + ), + min_quality: float = Query( + default=0.0, ge=0.0, le=1.0, description="Minimum quality score" + ), + limit: int = Query(default=50, ge=1, le=200, description="Page size"), + offset: int = Query(default=0, ge=0, description="Pagination offset"), + ) -> FAQFeedResponse: + """Public, read-only FAQ feed for this community. + + Returns synthesized question/answer entries generated from the + community's mailing-list and forum archives. Disabled by default; + a community opts in via ``public_feeds.faq: true`` in its config. + Email addresses are redacted from the output. ``total`` is the full + match count before pagination, in both browse and search modes. + """ + config = info.community_config + if config is None or config.public_feeds is None or not config.public_feeds.faq: + raise HTTPException( + status_code=404, + detail="Public FAQ feed is not enabled for this community.", + ) + + try: + entries, total = list_faq_entries( + project=community_id, + limit=limit, + offset=offset, + query=q, + category=category, + min_quality=min_quality, + ) + except sqlite3.Error: + logger.exception("Failed to query FAQ feed for community %s", community_id) + raise HTTPException( + status_code=503, + detail="Knowledge database is temporarily unavailable.", + ) + except Exception: + logger.exception("Unexpected error serving FAQ feed for community %s", community_id) + raise HTTPException( + status_code=500, + detail="An unexpected error occurred while building the FAQ feed.", + ) + + # Public, read-only data; cacheable like the other /…/public endpoints. + response.headers["Cache-Control"] = "public, max-age=3600" + return FAQFeedResponse( + community_id=community_id, + total=total, + limit=limit, + offset=offset, + entries=[_faq_result_to_response(e) for e in entries], + ) + + @router.get("/citations", response_model=CitationsFeedResponse) + async def community_citations(response: Response) -> CitationsFeedResponse: + """Public, read-only citation dashboard for this community. + + Returns per-year counts of papers citing the community's canonical + works, plus a stacked breakdown keyed by the cited DOI (the shape + behind a citations-per-year chart). Disabled by default; a community + opts in via ``public_feeds.citations: true`` in its config. + """ + config = info.community_config + if config is None or config.public_feeds is None or not config.public_feeds.citations: + raise HTTPException( + status_code=404, + detail="Public citations feed is not enabled for this community.", + ) + + try: + stats = get_citation_stats(project=community_id) + except sqlite3.Error: + logger.exception("Failed to query citations for community %s", community_id) + raise HTTPException( + status_code=503, + detail="Knowledge database is temporarily unavailable.", + ) + except Exception: + logger.exception( + "Unexpected error serving citations feed for community %s", community_id + ) + raise HTTPException( + status_code=500, + detail="An unexpected error occurred while building the citations feed.", + ) + + canonical_dois = list(config.citations.dois) if config.citations else [] + + response.headers["Cache-Control"] = "public, max-age=3600" + return CitationsFeedResponse( + community_id=community_id, + total=stats.total, + per_year=stats.per_year, + by_paper=stats.by_paper, + canonical_dois=canonical_dois, + ) + return router diff --git a/src/core/config/community.py b/src/core/config/community.py index 75d01b4..24bd3a2 100644 --- a/src/core/config/community.py +++ b/src/core/config/community.py @@ -637,6 +637,23 @@ def validate_agent_roles(self) -> "FAQGenerationConfig": return self +class PublicFeedsConfig(BaseModel): + """Opt-in flags for exposing community data as public, read-only JSON feeds. + + Both feeds are off by default. Enabling a feed publishes already-synced + data (FAQ entries, citation counts) at unauthenticated endpoints so + communities can build their own frontends on top of it. + """ + + model_config = ConfigDict(extra="forbid") + + faq: bool = False + """Expose generated FAQ entries at GET /{community_id}/faq.""" + + citations: bool = False + """Expose canonical-paper citation counts at GET /{community_id}/citations.""" + + class BudgetConfig(BaseModel): """Budget limits and alert thresholds for a community. @@ -918,6 +935,9 @@ def validate_id(cls, v: str) -> str: faq_generation: FAQGenerationConfig | None = None """FAQ generation configuration from threaded discussions (mailman, discourse, etc.).""" + public_feeds: PublicFeedsConfig | None = None + """Opt-in flags for exposing FAQ/citation data as public JSON feeds.""" + sync: SyncConfig | None = None """Per-community sync schedule configuration. diff --git a/src/knowledge/db.py b/src/knowledge/db.py index 5c9166d..ba2dfb6 100644 --- a/src/knowledge/db.py +++ b/src/knowledge/db.py @@ -132,6 +132,9 @@ def active_mirror_context(mirror_id: str) -> Iterator[None]: url TEXT NOT NULL, created_at TEXT, synced_at TEXT NOT NULL, + -- Canonical DOI this paper cites, when discovered via citation sync. + -- NULL for papers found through keyword search rather than a citation link. + cites_doi TEXT, UNIQUE(source, external_id) ); @@ -409,6 +412,8 @@ def active_mirror_context(mirror_id: str) -> Iterator[None]: CREATE INDEX IF NOT EXISTS idx_github_items_status ON github_items(status); CREATE INDEX IF NOT EXISTS idx_github_items_type ON github_items(item_type); CREATE INDEX IF NOT EXISTS idx_papers_source ON papers(source); +-- idx_papers_cites_doi is created in _migrate_db, after the cites_doi column +-- is ensured, so init_db stays safe on databases predating that column. CREATE INDEX IF NOT EXISTS idx_docstrings_repo ON docstrings(repo); CREATE INDEX IF NOT EXISTS idx_docstrings_language ON docstrings(language); CREATE INDEX IF NOT EXISTS idx_messages_list ON mailing_list_messages(list_name); @@ -507,6 +512,28 @@ def _migrate_db(conn: sqlite3.Connection) -> None: # Table doesn't exist yet - this is fine, schema will create it logger.debug("Docstrings table not found during migration (will be created): %s", e) + # Migration: Add cites_doi column to papers table (added 2026-06-09). + # The index lives here (not in SCHEMA_SQL) so executescript never references + # cites_doi on a database created before the column existed. + try: + cursor = conn.execute("PRAGMA table_info(papers)") + columns = [row[1] for row in cursor.fetchall()] + except sqlite3.OperationalError as e: + # Only the PRAGMA is guarded here: a missing papers table is fine since + # SCHEMA_SQL creates it. DDL errors below (locked DB, I/O fault) must + # propagate rather than be swallowed and leave the table un-indexed. + logger.debug("Papers table not found during migration (will be created): %s", e) + columns = [] + + if columns: # papers table exists; migrate it in place + if "cites_doi" not in columns: + logger.info("Migrating papers table: adding cites_doi column") + conn.execute("ALTER TABLE papers ADD COLUMN cites_doi TEXT") + logger.info("Migration complete: cites_doi column added to papers") + # Ensure the index exists for both new and migrated databases. + conn.execute("CREATE INDEX IF NOT EXISTS idx_papers_cites_doi ON papers(cites_doi)") + conn.commit() + def init_db(project: str = "hed") -> None: """Initialize database schema for a project. @@ -586,6 +613,7 @@ def upsert_paper( first_message: str | None, url: str, created_at: str | None, + cites_doi: str | None = None, ) -> None: """Insert or update a paper. @@ -597,6 +625,14 @@ def upsert_paper( first_message: Abstract (limited to ~2000 chars) url: URL to the paper (DOI or source URL) created_at: Publication date (ISO 8601 or year string) + cites_doi: Canonical DOI this paper cites, when known from a citation + sync. ``None`` for keyword-search results. On conflict the first + recorded link is kept (COALESCE), so a later keyword sync passing + ``None`` never erases an existing citation link, and a re-sync + backfills the link onto rows stored before this column existed. + A single column holds one link: a paper citing two tracked DOIs is + attributed to whichever was synced first (it is still counted once + in the per-year total, only its by-paper bucket is approximate). """ # Limit first_message size if first_message and len(first_message) > 2000: @@ -605,14 +641,15 @@ def upsert_paper( conn.execute( """ INSERT INTO papers (source, external_id, title, first_message, - status, url, created_at, synced_at) - VALUES (?, ?, ?, ?, 'published', ?, ?, ?) + status, url, created_at, synced_at, cites_doi) + VALUES (?, ?, ?, ?, 'published', ?, ?, ?, ?) ON CONFLICT(source, external_id) DO UPDATE SET title=excluded.title, first_message=excluded.first_message, - synced_at=excluded.synced_at + synced_at=excluded.synced_at, + cites_doi=COALESCE(papers.cites_doi, excluded.cites_doi) """, - (source, external_id, title, first_message, url, created_at, _now_iso()), + (source, external_id, title, first_message, url, created_at, _now_iso(), cites_doi), ) diff --git a/src/knowledge/papers_sync.py b/src/knowledge/papers_sync.py index a83806b..f185e27 100644 --- a/src/knowledge/papers_sync.py +++ b/src/knowledge/papers_sync.py @@ -158,6 +158,7 @@ def _store_papers( project: str, *, force_source: str | None = None, + cites_doi: str | None = None, ) -> dict[str, int]: """Upsert opencite papers into the knowledge DB, returning counts by source. @@ -167,6 +168,8 @@ def _store_papers( force_source: When set (a single-source sync), record this OSA source label using its native identifier; falls back to the priority mapping if that identifier is missing. + cites_doi: Canonical DOI these papers cite, recorded on each row when + storing the results of a citation sync. ``None`` for keyword search. """ counts: dict[str, int] = {} with get_connection(project) as conn: @@ -193,6 +196,7 @@ def _store_papers( first_message=paper.abstract or None, url=_paper_url(paper), created_at=paper.publication_date or (str(paper.year) if paper.year else None), + cites_doi=cites_doi, ) counts[source] = counts.get(source, 0) + 1 conn.commit() @@ -420,7 +424,7 @@ def sync_citing_papers( total = 0 for doi, papers in cited: try: - counts = _store_papers(papers, project) + counts = _store_papers(papers, project, cites_doi=doi) count = sum(counts.values()) update_sync_metadata("papers", f"citing_{doi}", count, project) logger.info("Synced %d papers citing %s", count, doi) diff --git a/src/knowledge/search.py b/src/knowledge/search.py index 61563f2..c8d0b7a 100644 --- a/src/knowledge/search.py +++ b/src/knowledge/search.py @@ -376,6 +376,77 @@ def search_github_items( return results +@dataclass +class CitationStats: + """Aggregated citation counts for a community's canonical papers.""" + + total: int + """Total citing papers with a recorded canonical link and a valid year.""" + + per_year: dict[str, int] + """Citing-paper count per publication year, summed across canonical DOIs.""" + + by_paper: dict[str, dict[str, int]] + """Per canonical DOI: a mapping of publication year to citing-paper count.""" + + +def get_citation_stats(project: str = "eeglab") -> CitationStats: + """Aggregate citation counts for the public citations dashboard. + + Counts papers that cite a community's canonical DOIs (``papers.cites_doi`` + is set), grouped by the citing paper's publication year. The year is the + leading four digits of ``created_at`` (ISO date or bare year); rows whose + ``created_at`` is missing or not a four-digit year are skipped so a bad + date never lands in a bogus year bucket. + + Args: + project: Community ID for database isolation. Defaults to 'eeglab'. + + Returns: + CitationStats with the overall ``total``, ``per_year`` totals, and the + stacked ``by_paper`` breakdown (canonical DOI -> year -> count). Years + are sorted ascending in every mapping. + """ + sql = """ + SELECT cites_doi, substr(created_at, 1, 4) AS yr, COUNT(*) AS cnt + FROM papers + WHERE cites_doi IS NOT NULL + AND created_at IS NOT NULL + AND substr(created_at, 1, 4) GLOB '[0-9][0-9][0-9][0-9]' + GROUP BY cites_doi, yr + """ + + per_year: dict[str, int] = {} + by_paper: dict[str, dict[str, int]] = {} + total = 0 + try: + with get_connection(project) as conn: + for row in conn.execute(sql): + doi = row["cites_doi"] + year = row["yr"] + count = row["cnt"] + per_year[year] = per_year.get(year, 0) + count + by_paper.setdefault(doi, {})[year] = count + total += count + except sqlite3.OperationalError as e: + logger.error( + "Database operational error computing citation stats: %s", + e, + exc_info=True, + extra={"project": project}, + ) + raise + except sqlite3.Error as e: + logger.warning("Database error computing citation stats (project=%s): %s", project, e) + raise + + return CitationStats( + total=total, + per_year=dict(sorted(per_year.items())), + by_paper={doi: dict(sorted(years.items())) for doi, years in by_paper.items()}, + ) + + def search_papers( query: str, project: str = "hed", @@ -792,6 +863,28 @@ class FAQResult: first_message_date: str +def _parse_faq_tags(raw: str | None, *, thread_url: str, project: str) -> list[str]: + """Decode a FAQ entry's JSON ``tags`` column, tolerating malformed data. + + The column is written by the summarizer as a JSON array. A corrupt value + should degrade to an empty tag list (and a warning) rather than raise a + ``JSONDecodeError`` that escapes the sqlite handlers and surfaces as an + unlogged 500 at the API layer. + """ + if not raw: + return [] + try: + return json.loads(raw) + except (json.JSONDecodeError, TypeError): + logger.warning( + "Invalid JSON in FAQ tags (thread_url=%s, project=%s): %r", + thread_url, + project, + raw, + ) + return [] + + def search_faq_entries( query: str, project: str = "eeglab", @@ -845,7 +938,7 @@ def search_faq_entries( params[0] = safe_query for row in conn.execute(sql, params): - tags = json.loads(row["tags"]) if row["tags"] else [] + tags = _parse_faq_tags(row["tags"], thread_url=row["thread_url"], project=project) results.append( FAQResult( @@ -876,6 +969,111 @@ def search_faq_entries( return results +def list_faq_entries( + project: str = "eeglab", + limit: int = 50, + offset: int = 0, + query: str | None = None, + list_name: str | None = None, + category: str | None = None, + min_quality: float = 0.0, +) -> tuple[list[FAQResult], int]: + """List FAQ entries for the public feed, with pagination metadata. + + Serves both browse mode (no ``query``) and search mode (``query`` set, via + FTS5). Unlike :func:`search_faq_entries`, this always returns the full + matching ``total`` count computed before LIMIT/OFFSET, so callers can + paginate correctly in either mode. + + Args: + project: Community ID for database isolation. Defaults to 'eeglab'. + limit: Maximum number of entries to return. + offset: Number of entries to skip (for pagination). + query: Optional full-text search phrase. When omitted, all entries + matching the filters are browsed, ordered by quality then recency. + list_name: Filter by mailing list name. + category: Filter by category (e.g., 'troubleshooting', 'how-to'). + min_quality: Minimum quality score (0.0-1.0). + + Returns: + Tuple of (entries, total_count) where total_count is the number of + entries matching the query and filters before limit/offset are applied. + """ + use_fts = bool(query and query.strip()) + + leading_params: list[str | int | float] = [] + if use_fts: + from_clause = "faq_entries_fts fts JOIN faq_entries f ON fts.rowid = f.id" + where_clause = "faq_entries_fts MATCH ?" + order_clause = "f.quality_score DESC, rank" + # Sanitize to prevent FTS5 injection (query is guaranteed non-None here). + leading_params.append(_sanitize_fts5_query(query)) # type: ignore[arg-type] + else: + from_clause = "faq_entries f" + where_clause = "1=1" + order_clause = "f.quality_score DESC, f.first_message_date DESC" + + filters = "" + filter_params: list[str | int | float] = [] + if list_name: + filters += " AND f.list_name = ?" + filter_params.append(list_name) + if category: + filters += " AND f.category = ?" + filter_params.append(category) + if min_quality > 0: + filters += " AND f.quality_score >= ?" + filter_params.append(min_quality) + + base_params = [*leading_params, *filter_params] + count_sql = f"SELECT COUNT(*) FROM {from_clause} WHERE {where_clause}{filters}" + rows_sql = ( + "SELECT f.question, f.answer, f.thread_url, f.tags, f.category, " + "f.quality_score, f.message_count, f.first_message_date " + f"FROM {from_clause} WHERE {where_clause}{filters} " + f"ORDER BY {order_clause} LIMIT ? OFFSET ?" + ) + + results: list[FAQResult] = [] + try: + with get_connection(project) as conn: + total = conn.execute(count_sql, base_params).fetchone()[0] + + for row in conn.execute(rows_sql, [*base_params, limit, offset]): + tags = _parse_faq_tags(row["tags"], thread_url=row["thread_url"], project=project) + results.append( + FAQResult( + question=row["question"], + answer=row["answer"], + thread_url=row["thread_url"], + tags=tags, + category=row["category"], + quality_score=row["quality_score"], + message_count=row["message_count"], + first_message_date=row["first_message_date"] or "", + ) + ) + except sqlite3.OperationalError as e: + logger.error( + "Database operational error listing FAQ entries: %s", + e, + exc_info=True, + extra={"project": project}, + ) + raise + except sqlite3.Error as e: + logger.warning( + "Database error listing FAQ entries (project=%s, limit=%d, offset=%d): %s", + project, + limit, + offset, + e, + ) + raise + + return results, total + + @dataclass class BEPResult: """A BEP search result from the knowledge database.""" diff --git a/tests/test_api/test_citations_feed.py b/tests/test_api/test_citations_feed.py new file mode 100644 index 0000000..bbf0e6b --- /dev/null +++ b/tests/test_api/test_citations_feed.py @@ -0,0 +1,204 @@ +"""Tests for the public citations feed endpoint: GET /{community_id}/citations. + +Uses a real registered community, a temporary SQLite knowledge database with +citing papers, and the config gate toggled per test. No business logic is +mocked except in TestCitationsFeedErrors, where get_citation_stats is patched +at the router call boundary to inject DB/unexpected errors and verify the +503/500 responses. +""" + +import sqlite3 +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.api.routers.community import create_community_router +from src.assistants import discover_assistants, registry +from src.core.config.community import PublicFeedsConfig +from src.knowledge.db import get_connection, init_db, upsert_paper + +COMMUNITY_ID = "eeglab" +DOI_A = "10.1016/j.jneumeth.2003.10.009" +DOI_B = "10.1016/j.neuroimage.2019.05.026" + +discover_assistants() + + +@pytest.fixture +def citations_db(tmp_path: Path) -> Iterator[Path]: + """Temp knowledge DB with citing papers across two canonical DOIs.""" + db_path = tmp_path / "knowledge" / "test.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db(COMMUNITY_ID) + with get_connection(COMMUNITY_ID) as conn: + rows = [ + ("a1", "2019-05-01", DOI_A), + ("a2", "2019-11-20", DOI_A), + ("a3", "2020", DOI_A), + ("b1", "2020-02-02", DOI_B), + ("k1", "2021", None), # keyword-only, excluded from stats + ] + for external_id, created_at, cites_doi in rows: + upsert_paper( + conn, + source="openalex", + external_id=external_id, + title=f"Paper {external_id}", + first_message=None, + url=f"https://doi.org/10.test/{external_id}", + created_at=created_at, + cites_doi=cites_doi, + ) + conn.commit() + yield db_path + + +@pytest.fixture +def citations_enabled() -> Iterator[None]: + info = registry.get(COMMUNITY_ID) + assert info is not None and info.community_config is not None + original = info.community_config.public_feeds + info.community_config.public_feeds = PublicFeedsConfig(citations=True) + try: + yield + finally: + info.community_config.public_feeds = original + + +@pytest.fixture +def citations_disabled_none() -> Iterator[None]: + info = registry.get(COMMUNITY_ID) + assert info is not None and info.community_config is not None + original = info.community_config.public_feeds + info.community_config.public_feeds = None + try: + yield + finally: + info.community_config.public_feeds = original + + +@pytest.fixture +def citations_flag_false() -> Iterator[None]: + info = registry.get(COMMUNITY_ID) + assert info is not None and info.community_config is not None + original = info.community_config.public_feeds + info.community_config.public_feeds = PublicFeedsConfig(citations=False) + try: + yield + finally: + info.community_config.public_feeds = original + + +@pytest.fixture +def citations_enabled_no_config() -> Iterator[None]: + """Feed enabled but the community has no citations config block.""" + info = registry.get(COMMUNITY_ID) + assert info is not None and info.community_config is not None + orig_feeds = info.community_config.public_feeds + orig_citations = info.community_config.citations + info.community_config.public_feeds = PublicFeedsConfig(citations=True) + info.community_config.citations = None + try: + yield + finally: + info.community_config.public_feeds = orig_feeds + info.community_config.citations = orig_citations + + +@pytest.fixture +def client() -> TestClient: + app = FastAPI() + app.include_router(create_community_router(COMMUNITY_ID)) + return TestClient(app) + + +class TestCitationsFeedGate: + """The endpoint is opt-in via public_feeds.citations.""" + + @pytest.mark.usefixtures("citations_disabled_none") + def test_disabled_when_public_feeds_none(self, client, citations_db): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + resp = client.get(f"/{COMMUNITY_ID}/citations") + assert resp.status_code == 404 + + @pytest.mark.usefixtures("citations_flag_false") + def test_disabled_when_flag_false(self, client, citations_db): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + resp = client.get(f"/{COMMUNITY_ID}/citations") + assert resp.status_code == 404 + + @pytest.mark.usefixtures("citations_enabled") + def test_enabled_returns_200(self, client, citations_db): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + resp = client.get(f"/{COMMUNITY_ID}/citations") + assert resp.status_code == 200 + + +@pytest.mark.usefixtures("citations_enabled") +class TestCitationsFeedContent: + def test_total_and_per_year(self, client, citations_db): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + resp = client.get(f"/{COMMUNITY_ID}/citations") + body = resp.json() + assert body["community_id"] == COMMUNITY_ID + assert body["total"] == 4 # a1,a2,a3,b1 ; k1 unlinked excluded + assert body["per_year"] == {"2019": 2, "2020": 2} + + def test_by_paper_stacked_breakdown(self, client, citations_db): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + resp = client.get(f"/{COMMUNITY_ID}/citations") + by_paper = resp.json()["by_paper"] + assert by_paper == { + DOI_A: {"2019": 2, "2020": 1}, + DOI_B: {"2020": 1}, + } + + def test_canonical_dois_from_config(self, client, citations_db): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + resp = client.get(f"/{COMMUNITY_ID}/citations") + canonical = resp.json()["canonical_dois"] + # eeglab config tracks these canonical DOIs. + assert DOI_A in canonical + assert DOI_B in canonical + + def test_cache_control_header(self, client, citations_db): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + resp = client.get(f"/{COMMUNITY_ID}/citations") + assert resp.headers["Cache-Control"] == "public, max-age=3600" + + +class TestCitationsFeedNoConfig: + """Feed enabled for a community without a citations config block.""" + + @pytest.mark.usefixtures("citations_enabled_no_config") + def test_canonical_dois_empty_when_no_citations_config(self, client, citations_db): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + resp = client.get(f"/{COMMUNITY_ID}/citations") + body = resp.json() + assert resp.status_code == 200 + assert body["canonical_dois"] == [] + # Stats still come from the DB regardless of config presence. + assert body["total"] == 4 + + +@pytest.mark.usefixtures("citations_enabled") +class TestCitationsFeedErrors: + def test_db_error_returns_503(self, client): + with patch( + "src.api.routers.community.get_citation_stats", + side_effect=sqlite3.OperationalError("db is locked"), + ): + resp = client.get(f"/{COMMUNITY_ID}/citations") + assert resp.status_code == 503 + + def test_unexpected_error_returns_500(self, client): + with patch( + "src.api.routers.community.get_citation_stats", + side_effect=RuntimeError("boom"), + ): + resp = client.get(f"/{COMMUNITY_ID}/citations") + assert resp.status_code == 500 diff --git a/tests/test_api/test_faq_feed.py b/tests/test_api/test_faq_feed.py new file mode 100644 index 0000000..9408a9e --- /dev/null +++ b/tests/test_api/test_faq_feed.py @@ -0,0 +1,256 @@ +"""Tests for the public FAQ feed endpoint: GET /{community_id}/faq. + +Uses a real registered community, a temporary SQLite knowledge database +populated with FAQ rows, and the config gate toggled per test. No business +logic is mocked; only the database path and the opt-in flag are controlled. +""" + +import sqlite3 +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.api.routers.community import create_community_router +from src.assistants import discover_assistants, registry +from src.core.config.community import PublicFeedsConfig +from src.knowledge.db import get_connection, init_db, upsert_faq_entry + +COMMUNITY_ID = "eeglab" + +discover_assistants() + + +@pytest.fixture +def faq_db(tmp_path: Path) -> Iterator[Path]: + """Temp knowledge DB populated with FAQ entries, including one with an email.""" + db_path = tmp_path / "knowledge" / "test.db" + # Write through the same project the endpoint reads (COMMUNITY_ID) so the + # test does not rely on get_db_path ignoring its project argument. + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db(COMMUNITY_ID) + with get_connection(COMMUNITY_ID) as conn: + upsert_faq_entry( + conn, + list_name="eeglablist", + thread_id="t1", + thread_url="https://example.org/t1", + question="How do I run ICA in EEGLAB?", + answer="Use runica from the Tools menu.", + tags=["ica"], + category="how-to", + message_count=3, + participant_count=2, + first_message_date="2020-01-01", + quality_score=0.95, + summary_model="test-model", + ) + # t2 carries an email in the question, the answer, and a tag so the + # endpoint's redaction can be verified across all three fields. + upsert_faq_entry( + conn, + list_name="eeglablist", + thread_id="t2", + thread_url="https://example.org/t2", + question="Who do I contact (e.g. sales@brainproducts.com) for support?", + answer="Email support@brainproducts.com for hardware questions.", + tags=["hardware", "contact:info@vendor.com"], + category="reference", + message_count=2, + participant_count=2, + first_message_date="2021-01-01", + quality_score=0.70, + summary_model="test-model", + ) + conn.commit() + yield db_path + + +@pytest.fixture +def feeds_enabled() -> Iterator[None]: + """Enable public_feeds.faq on the community config, restoring it afterward.""" + info = registry.get(COMMUNITY_ID) + assert info is not None and info.community_config is not None + original = info.community_config.public_feeds + info.community_config.public_feeds = PublicFeedsConfig(faq=True) + try: + yield + finally: + info.community_config.public_feeds = original + + +@pytest.fixture +def feeds_disabled() -> Iterator[None]: + """Force public_feeds off (None), restoring the original afterward.""" + info = registry.get(COMMUNITY_ID) + assert info is not None and info.community_config is not None + original = info.community_config.public_feeds + info.community_config.public_feeds = None + try: + yield + finally: + info.community_config.public_feeds = original + + +@pytest.fixture +def feeds_faq_false() -> Iterator[None]: + """public_feeds present but faq disabled (the non-None gate branch).""" + info = registry.get(COMMUNITY_ID) + assert info is not None and info.community_config is not None + original = info.community_config.public_feeds + info.community_config.public_feeds = PublicFeedsConfig(faq=False) + try: + yield + finally: + info.community_config.public_feeds = original + + +@pytest.fixture +def client() -> TestClient: + app = FastAPI() + app.include_router(create_community_router(COMMUNITY_ID)) + return TestClient(app) + + +class TestFAQFeedGate: + """The endpoint is opt-in via public_feeds.faq.""" + + @pytest.mark.usefixtures("feeds_disabled") + def test_disabled_when_public_feeds_none(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq") + assert resp.status_code == 404 + + @pytest.mark.usefixtures("feeds_faq_false") + def test_disabled_when_faq_flag_false(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq") + assert resp.status_code == 404 + + @pytest.mark.usefixtures("feeds_enabled") + def test_enabled_returns_200(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq") + assert resp.status_code == 200 + + +@pytest.mark.usefixtures("feeds_enabled") +class TestFAQFeedContent: + """Response shape and filtering when enabled.""" + + def test_returns_all_entries(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq") + body = resp.json() + assert body["community_id"] == COMMUNITY_ID + assert body["total"] == 2 + assert len(body["entries"]) == 2 + # Ordered by quality descending + assert body["entries"][0]["quality_score"] == 0.95 + + def test_exposed_fields_only(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq") + entry = resp.json()["entries"][0] + assert set(entry.keys()) == { + "question", + "answer", + "tags", + "category", + "quality_score", + "message_count", + "first_message_date", + "thread_url", + } + + def test_emails_are_redacted(self, client, faq_db): + """Emails are stripped from question, answer, and tags alike.""" + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq") + entries = resp.json()["entries"] + blob = " ".join( + e["question"] + " " + e["answer"] + " " + " ".join(e["tags"]) for e in entries + ) + assert "support@brainproducts.com" not in blob + assert "sales@brainproducts.com" not in blob + assert "info@vendor.com" not in blob + assert "[email redacted]" in blob + # Redaction reached all three field types on the t2 entry. + t2 = next(e for e in entries if e["category"] == "reference") + assert "[email redacted]" in t2["question"] + assert "[email redacted]" in t2["answer"] + assert any("[email redacted]" in tag for tag in t2["tags"]) + + def test_category_filter(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq", params={"category": "how-to"}) + body = resp.json() + assert body["total"] == 1 + assert body["entries"][0]["category"] == "how-to" + + def test_min_quality_filter(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq", params={"min_quality": 0.9}) + body = resp.json() + assert body["total"] == 1 + assert body["entries"][0]["quality_score"] >= 0.9 + + def test_search_query(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq", params={"q": "ICA"}) + body = resp.json() + # Only the t1 entry mentions ICA; total is the real match count. + assert body["total"] == 1 + assert len(body["entries"]) == 1 + assert "ICA" in body["entries"][0]["question"] + + def test_pagination(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq", params={"limit": 1, "offset": 0}) + body = resp.json() + assert body["total"] == 2 + assert len(body["entries"]) == 1 + assert body["limit"] == 1 + assert body["offset"] == 0 + + def test_cache_control_header(self, client, faq_db): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + resp = client.get(f"/{COMMUNITY_ID}/faq") + assert resp.headers["Cache-Control"] == "public, max-age=3600" + + +@pytest.mark.usefixtures("feeds_enabled", "faq_db") +class TestFAQFeedValidation: + """Query parameter bounds are enforced (rejected before DB access).""" + + def test_invalid_min_quality_rejected(self, client): + resp = client.get(f"/{COMMUNITY_ID}/faq", params={"min_quality": 5}) + assert resp.status_code == 422 + + def test_limit_upper_bound_enforced(self, client): + resp = client.get(f"/{COMMUNITY_ID}/faq", params={"limit": 9999}) + assert resp.status_code == 422 + + +@pytest.mark.usefixtures("feeds_enabled") +class TestFAQFeedErrors: + """Database failures surface as 503, not silent empty responses.""" + + def test_browse_db_error_returns_503(self, client): + with patch( + "src.api.routers.community.list_faq_entries", + side_effect=sqlite3.OperationalError("db is locked"), + ): + resp = client.get(f"/{COMMUNITY_ID}/faq") + assert resp.status_code == 503 + + def test_search_db_error_returns_503(self, client): + with patch( + "src.api.routers.community.list_faq_entries", + side_effect=sqlite3.OperationalError("db is locked"), + ): + resp = client.get(f"/{COMMUNITY_ID}/faq", params={"q": "ICA"}) + assert resp.status_code == 503 diff --git a/tests/test_knowledge/test_citation_stats.py b/tests/test_knowledge/test_citation_stats.py new file mode 100644 index 0000000..4d828cb --- /dev/null +++ b/tests/test_knowledge/test_citation_stats.py @@ -0,0 +1,179 @@ +"""Tests for citation stats aggregation and the cites_doi linkage column. + +Uses a real temporary SQLite database (only the DB path is redirected); no +business logic is mocked. +""" + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from src.knowledge.db import get_connection, init_db, upsert_paper +from src.knowledge.search import CitationStats, get_citation_stats + +DOI_A = "10.1016/j.jneumeth.2003.10.009" +DOI_B = "10.1016/j.neuroimage.2019.05.026" + + +def _add_paper(conn, external_id, *, created_at, cites_doi=None, source="openalex"): + upsert_paper( + conn, + source=source, + external_id=external_id, + title=f"Citing paper {external_id}", + first_message=None, + url=f"https://doi.org/10.test/{external_id}", + created_at=created_at, + cites_doi=cites_doi, + ) + + +@pytest.fixture +def citations_db(tmp_path: Path): + """Temp DB with citing papers across two canonical DOIs and several years.""" + db_path = tmp_path / "knowledge" / "test.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + with get_connection() as conn: + # DOI_A: 2 in 2019, 1 in 2020 + _add_paper(conn, "a1", created_at="2019-05-01", cites_doi=DOI_A) + _add_paper(conn, "a2", created_at="2019-11-20", cites_doi=DOI_A) + _add_paper(conn, "a3", created_at="2020", cites_doi=DOI_A) + # DOI_B: 1 in 2020, 1 in 2021 + _add_paper(conn, "b1", created_at="2020-02-02", cites_doi=DOI_B) + _add_paper(conn, "b2", created_at="2021-07-07", cites_doi=DOI_B) + # Keyword-search paper (no citation link) - excluded from stats + _add_paper(conn, "k1", created_at="2022", cites_doi=None) + # Citing paper with an unusable date - excluded from year buckets + _add_paper(conn, "x1", created_at="", cites_doi=DOI_A) + _add_paper(conn, "x2", created_at=None, cites_doi=DOI_B) + conn.commit() + yield db_path + + +class TestGetCitationStats: + def test_returns_citation_stats_object(self, citations_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + stats = get_citation_stats(project="eeglab") + assert isinstance(stats, CitationStats) + + def test_total_excludes_unlinked_and_undated(self, citations_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + stats = get_citation_stats(project="eeglab") + # 5 linked papers with valid years (a1,a2,a3,b1,b2); k1 unlinked, + # x1/x2 undated are excluded. + assert stats.total == 5 + + def test_per_year_aggregates_across_dois(self, citations_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + stats = get_citation_stats(project="eeglab") + assert stats.per_year == {"2019": 2, "2020": 2, "2021": 1} + + def test_per_year_is_sorted_ascending(self, citations_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + stats = get_citation_stats(project="eeglab") + assert list(stats.per_year.keys()) == sorted(stats.per_year.keys()) + + def test_by_paper_stacked_breakdown(self, citations_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=citations_db): + stats = get_citation_stats(project="eeglab") + assert stats.by_paper == { + DOI_A: {"2019": 2, "2020": 1}, + DOI_B: {"2020": 1, "2021": 1}, + } + + def test_empty_database(self, tmp_path: Path): + db_path = tmp_path / "knowledge" / "empty.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + stats = get_citation_stats(project="eeglab") + assert stats.total == 0 + assert stats.per_year == {} + assert stats.by_paper == {} + + +class TestCitesDoiUpsert: + def test_backfill_sets_link_on_existing_row(self, tmp_path: Path): + """A row first stored without a link gets it on a later citation sync.""" + db_path = tmp_path / "knowledge" / "test.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + with get_connection() as conn: + _add_paper(conn, "p1", created_at="2020", cites_doi=None) + _add_paper(conn, "p1", created_at="2020", cites_doi=DOI_A) + conn.commit() + row = conn.execute( + "SELECT cites_doi FROM papers WHERE external_id = 'p1'" + ).fetchone() + assert row["cites_doi"] == DOI_A + + def test_first_link_wins_over_later_link(self, tmp_path: Path): + """COALESCE keeps the first recorded canonical DOI for overlapping papers.""" + db_path = tmp_path / "knowledge" / "test.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + with get_connection() as conn: + _add_paper(conn, "p1", created_at="2020", cites_doi=DOI_A) + _add_paper(conn, "p1", created_at="2020", cites_doi=DOI_B) + conn.commit() + row = conn.execute( + "SELECT cites_doi FROM papers WHERE external_id = 'p1'" + ).fetchone() + assert row["cites_doi"] == DOI_A + + def test_keyword_sync_does_not_erase_link(self, tmp_path: Path): + """A later keyword sync (cites_doi=None) must not clobber an existing link.""" + db_path = tmp_path / "knowledge" / "test.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + with get_connection() as conn: + _add_paper(conn, "p1", created_at="2020", cites_doi=DOI_A) + _add_paper(conn, "p1", created_at="2020", cites_doi=None) + conn.commit() + row = conn.execute( + "SELECT cites_doi FROM papers WHERE external_id = 'p1'" + ).fetchone() + assert row["cites_doi"] == DOI_A + + +class TestCitesDoiMigration: + def test_migration_adds_column_to_legacy_papers_table(self, tmp_path: Path): + """A papers table created before cites_doi gains the column via init_db.""" + db_path = tmp_path / "knowledge" / "legacy.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + # Simulate a pre-migration schema: papers without cites_doi. + with get_connection() as conn: + conn.execute( + """ + CREATE TABLE papers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source TEXT NOT NULL, + external_id TEXT NOT NULL, + title TEXT NOT NULL, + first_message TEXT, + status TEXT NOT NULL DEFAULT 'published', + url TEXT NOT NULL, + created_at TEXT, + synced_at TEXT NOT NULL, + UNIQUE(source, external_id) + ) + """ + ) + conn.commit() + cols_before = [r[1] for r in conn.execute("PRAGMA table_info(papers)")] + assert "cites_doi" not in cols_before + + # Running init_db must migrate the existing table in place. + init_db() + with get_connection() as conn: + cols_after = [r[1] for r in conn.execute("PRAGMA table_info(papers)")] + # The new column is usable for inserts after migration. + _add_paper(conn, "p1", created_at="2020", cites_doi=DOI_A) + conn.commit() + row = conn.execute( + "SELECT cites_doi FROM papers WHERE external_id = 'p1'" + ).fetchone() + + assert "cites_doi" in cols_after + assert row["cites_doi"] == DOI_A diff --git a/tests/test_knowledge/test_faq_feed.py b/tests/test_knowledge/test_faq_feed.py new file mode 100644 index 0000000..e1436f9 --- /dev/null +++ b/tests/test_knowledge/test_faq_feed.py @@ -0,0 +1,213 @@ +"""Tests for the public FAQ feed listing helper. + +Uses a temporary SQLite database populated with real FAQ rows (no mocks of +business logic; only the database path is redirected to a temp file). +""" + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from src.knowledge.db import get_connection, init_db, upsert_faq_entry +from src.knowledge.search import FAQResult, list_faq_entries + + +@pytest.fixture +def faq_db(tmp_path: Path): + """Create a test database populated with FAQ entries.""" + db_path = tmp_path / "knowledge" / "test.db" + + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + + with get_connection() as conn: + entries = [ + { + "thread_id": "t1", + "question": "How do I run ICA in EEGLAB?", + "answer": "Use runica via the Tools menu.", + "tags": ["ica", "eeglab"], + "category": "how-to", + "quality_score": 0.95, + "first_message_date": "2020-01-01", + }, + { + "thread_id": "t2", + "question": "Why does my dataset fail to load?", + "answer": "Check the file path and channel locations.", + "tags": ["loading"], + "category": "troubleshooting", + "quality_score": 0.80, + "first_message_date": "2021-06-15", + }, + { + "thread_id": "t3", + "question": "What is a reference electrode?", + "answer": "Contact support@brainproducts.com for hardware details.", + "tags": ["reference"], + "category": "reference", + "quality_score": 0.60, + "first_message_date": "2019-03-20", + }, + ] + for e in entries: + upsert_faq_entry( + conn, + list_name="eeglablist", + thread_id=e["thread_id"], + thread_url=f"https://example.org/{e['thread_id']}", + question=e["question"], + answer=e["answer"], + tags=e["tags"], + category=e["category"], + message_count=3, + participant_count=2, + first_message_date=e["first_message_date"], + quality_score=e["quality_score"], + summary_model="test-model", + ) + conn.commit() + + yield db_path + + +class TestListFAQEntries: + """Tests for list_faq_entries (browse mode, no FTS query).""" + + def test_returns_all_entries_and_total(self, faq_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + entries, total = list_faq_entries(project="eeglab") + + assert total == 3 + assert len(entries) == 3 + assert all(isinstance(e, FAQResult) for e in entries) + + def test_ordered_by_quality_descending(self, faq_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + entries, _ = list_faq_entries(project="eeglab") + + scores = [e.quality_score for e in entries] + assert scores == sorted(scores, reverse=True) + assert entries[0].quality_score == 0.95 + + def test_min_quality_filter(self, faq_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + entries, total = list_faq_entries(project="eeglab", min_quality=0.85) + + assert total == 1 + assert len(entries) == 1 + assert entries[0].quality_score >= 0.85 + + def test_category_filter(self, faq_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + entries, total = list_faq_entries(project="eeglab", category="troubleshooting") + + assert total == 1 + assert entries[0].category == "troubleshooting" + + def test_pagination_limit_and_offset(self, faq_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + page1, total1 = list_faq_entries(project="eeglab", limit=2, offset=0) + page2, total2 = list_faq_entries(project="eeglab", limit=2, offset=2) + + # total is the full count regardless of pagination window + assert total1 == 3 + assert total2 == 3 + assert len(page1) == 2 + assert len(page2) == 1 + # No overlap between pages + page1_questions = {e.question for e in page1} + page2_questions = {e.question for e in page2} + assert page1_questions.isdisjoint(page2_questions) + + def test_empty_database_returns_zero(self, tmp_path: Path): + db_path = tmp_path / "knowledge" / "empty.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + entries, total = list_faq_entries(project="eeglab") + + assert total == 0 + assert entries == [] + + def test_list_name_filter(self, tmp_path: Path): + """list_name filter restricts results to a single mailing list.""" + db_path = tmp_path / "knowledge" / "lists.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + with get_connection() as conn: + for list_name, thread_id in [ + ("list-a", "a1"), + ("list-a", "a2"), + ("list-b", "b1"), + ]: + upsert_faq_entry( + conn, + list_name=list_name, + thread_id=thread_id, + thread_url=f"https://example.org/{thread_id}", + question=f"Question {thread_id}?", + answer="An answer.", + tags=["t"], + category="how-to", + message_count=2, + participant_count=2, + first_message_date="2020-01-01", + quality_score=0.8, + summary_model="test-model", + ) + conn.commit() + + entries, total = list_faq_entries(project="eeglab", list_name="list-a") + + assert total == 2 + assert len(entries) == 2 + assert {e.question for e in entries} == {"Question a1?", "Question a2?"} + + +class TestListFAQEntriesSearch: + """Search mode of list_faq_entries (query set, via FTS5).""" + + def test_query_matches_entries(self, faq_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + entries, total = list_faq_entries(project="eeglab", query="ICA") + + assert total >= 1 + assert any("ICA" in e.question for e in entries) + + def test_query_no_match_returns_empty(self, faq_db: Path): + with patch("src.knowledge.db.get_db_path", return_value=faq_db): + entries, total = list_faq_entries(project="eeglab", query="zzzznomatchterm") + + assert total == 0 + assert entries == [] + + def test_query_total_is_full_count_not_page_size(self, tmp_path: Path): + """total reflects all FTS matches, independent of the page limit.""" + db_path = tmp_path / "knowledge" / "search.db" + with patch("src.knowledge.db.get_db_path", return_value=db_path): + init_db() + with get_connection() as conn: + for i in range(3): + upsert_faq_entry( + conn, + list_name="eeglablist", + thread_id=f"c{i}", + thread_url=f"https://example.org/c{i}", + question=f"How do I handle channels in case {i}?", + answer="Inspect the channel locations.", + tags=["channels"], + category="how-to", + message_count=2, + participant_count=2, + first_message_date="2020-01-01", + quality_score=0.8, + summary_model="test-model", + ) + conn.commit() + + page, total = list_faq_entries(project="eeglab", query="channels", limit=1) + + assert len(page) == 1 + assert total == 3 + assert total > len(page) diff --git a/tests/test_knowledge/test_papers_sync.py b/tests/test_knowledge/test_papers_sync.py index b23740c..edf45c3 100644 --- a/tests/test_knowledge/test_papers_sync.py +++ b/tests/test_knowledge/test_papers_sync.py @@ -165,6 +165,21 @@ def test_upsert_deduplicates_same_paper(self, temp_db: Path): count = conn.execute("SELECT COUNT(*) AS c FROM papers").fetchone()["c"] assert count == 1 + def test_stores_cites_doi_on_each_row(self, temp_db: Path): + # A citation sync threads the canonical DOI through to each stored row. + papers = [ + Paper(title="Citing A", ids=IDSet(openalex_id="https://openalex.org/W1"), year=2023), + Paper(title="Citing B", ids=IDSet(openalex_id="https://openalex.org/W2"), year=2024), + ] + with patch("src.knowledge.db.get_db_path", return_value=temp_db): + _store_papers(papers, "test", cites_doi="10.1/canonical") + with get_connection("test") as conn: + links = { + r["external_id"]: r["cites_doi"] + for r in conn.execute("SELECT external_id, cites_doi FROM papers") + } + assert links == {"W1": "10.1/canonical", "W2": "10.1/canonical"} + def test_force_source_uses_native_id(self, temp_db: Path): # A PubMed-restricted sync should label the row 'pubmed' using the PMID, # even though the paper also carries an OpenAlex id.