diff --git a/ops/active_monitor.py b/ops/active_monitor.py index 57e9dee2..3473d7e3 100644 --- a/ops/active_monitor.py +++ b/ops/active_monitor.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Iterable +import logging import os import re @@ -20,12 +21,14 @@ from scripts.process_curve_candidates import DEFAULT_AUTOTRADER_SOURCE, load_autotrader_market from shared.canonical_tagging import UNCLASSIFIED, is_canonical_eligible from shared.comps_engine import parse_currency, parse_numeric +from shared.csv_utils import CSV_READ_ERRORS from shared.curves import interpolate_base_by_year, list_curve_tags, load_curves, resolve_curve_canonical_tag from shared.data_loader import dataset_path from shared.location_utils import extract_state from shared.repair_pricing import assess_repairs, repair_fragments_to_records, vehicle_class_for_listing from shared.repair_review import append_live_review_items +logger = logging.getLogger(__name__) ACTIVE_RESTRICTED_PATH = dataset_path("active_vehicle_details_restricted.csv") ACTIVE_LIVE_PATH = dataset_path("active_vehicle_details.csv") @@ -161,7 +164,13 @@ def _load_normalized_conditions() -> pd.DataFrame: return pd.DataFrame() try: df = pd.read_csv(NORMALIZED_CONDITIONS_PATH) - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable normalized conditions %s (%s: %s); condition context will be missing.", + NORMALIZED_CONDITIONS_PATH, + type(exc).__name__, + exc, + ) return pd.DataFrame() if "url" not in df.columns or "component_normalized" not in df.columns: return pd.DataFrame() diff --git a/pages/14_CURVE_PIPELINE.py b/pages/14_CURVE_PIPELINE.py index 56314071..4f205173 100644 --- a/pages/14_CURVE_PIPELINE.py +++ b/pages/14_CURVE_PIPELINE.py @@ -15,6 +15,7 @@ run_autotrader_scrape, update_autotrader_queue_status, ) +from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path from shared.styling import display_banner, hero_action_card, inject_global_styles, page_intro, section_heading @@ -46,7 +47,8 @@ def _load_csv(path: Path) -> pd.DataFrame: return pd.DataFrame() try: return pd.read_csv(path, low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + st.warning(f"Could not read {path}: {type(exc).__name__}: {exc}") return pd.DataFrame() diff --git a/pages/18_REPAIR_REVIEW.py b/pages/18_REPAIR_REVIEW.py index 10b6311d..7ffa99a4 100644 --- a/pages/18_REPAIR_REVIEW.py +++ b/pages/18_REPAIR_REVIEW.py @@ -8,6 +8,7 @@ import streamlit as st from shared.navigation import render_sidebar_navigation +from shared.csv_utils import CSV_READ_ERRORS from shared.repair_ai_classifier import AI_SUGGESTIONS_PATH, load_ai_suggestions from shared.repair_review import LIVE_QUEUE_PATH from shared.styling import clean_html, display_banner, escape_html, inject_global_styles, page_intro @@ -113,7 +114,8 @@ def load_decisions() -> pd.DataFrame: return pd.DataFrame(columns=REVIEW_COLUMNS) try: df = pd.read_csv(DECISIONS_PATH).fillna("") - except Exception: + except CSV_READ_ERRORS as exc: + st.warning(f"Could not read {DECISIONS_PATH}: {type(exc).__name__}: {exc}") return pd.DataFrame(columns=REVIEW_COLUMNS) for column in REVIEW_COLUMNS: if column not in df.columns: diff --git a/pages/3_ACTIVE_LISTINGS.py b/pages/3_ACTIVE_LISTINGS.py index af4ed197..71adf828 100644 --- a/pages/3_ACTIVE_LISTINGS.py +++ b/pages/3_ACTIVE_LISTINGS.py @@ -128,7 +128,12 @@ async def run_bid_update(links: list[str] | None = None, limit: int | None = Non target_links = target_links[:limit_arg] limit_arg = None with st.spinner("Updating bid and time data..."): - df, skipped_urls = await update_bids(input_links=target_links, limit=limit_arg) + try: + df, skipped_urls = await update_bids(input_links=target_links, limit=limit_arg) + except Exception as exc: # noqa: BLE001 - surface scraper failures in the UI + st.error(f"Bid update failed: {type(exc).__name__}: {exc}") + st.exception(exc) + return st.session_state.skipped_urls = skipped_urls if not df.empty: st.success(f"Updated {len(df)} listings in {CSV_FILE}.") diff --git a/project_memory/02_state/recent_changes.md b/project_memory/02_state/recent_changes.md index a419ff78..da44a2a4 100644 --- a/project_memory/02_state/recent_changes.md +++ b/project_memory/02_state/recent_changes.md @@ -1,5 +1,7 @@ # Recent Changes +- 2026-08-09: Made swallowed failures visible across the data-loading, scraping, scheduling, valuation, and Streamlit layers. Broad `except Exception` handlers that returned empty frames, `False`, or partial results were narrowed to the exceptions they actually expect (`CSV_READ_ERRORS`, `OSError`, `ValueError`, `yaml.YAMLError`, `ZoneInfoNotFoundError`) and now log or print the path plus exception before falling back, so an unreadable CSV, curve file, pricing schedule, alert log, or model artifact is reported instead of silently degrading a decision. Two behaviours changed deliberately: `update_bids()` now re-raises after its emergency snapshot instead of returning a partial frame as success (the Active Listings page surfaces the error), and a failed restricted-dataset build now aborts `update_master_database()` instead of printing and continuing. Remote data-bundle publication deliberately retains a broad UI-safe fallback, but now logs the exception details instead of failing silently. Added `tests/test_error_visibility.py`; full suite, Ruff, readiness, governance, and project-memory checks pass. + - 2026-08-13: Rebuilt dashboard authentication on current `main`. Every Streamlit entrypoint now fails closed until a password or PBKDF2 verifier is configured, with an explicit local-only opt-out; current VPS/development navigation separation is preserved, Autotrader controls accept only HTTPS `autotrader.com.au` targets, the devcontainer restores CORS/XSRF defaults, and focused authentication coverage accompanies the change. - 2026-08-13: Addressed the post-PR10 source review as an isolated hardening slice. Carsales/Apify imports now preserve source identity and zero values, normalize timestamps, deduplicate by ad ID and URL, validate paid exact URLs, and return a distinct deferred status; canonical tagging uses narrower body/series inference; AI Analysis restores diagnostic context with policy-aligned signal tones; atomic CSV locks verify live ownership; scheduler browser setup is bounded and failed daily runs remain catch-up eligible; merge-governance approval is restricted to the exact merged PR commit. diff --git a/scripts/ai_listing_valuation.py b/scripts/ai_listing_valuation.py index ab95c8f5..062e0eda 100644 --- a/scripts/ai_listing_valuation.py +++ b/scripts/ai_listing_valuation.py @@ -12,6 +12,7 @@ from scripts.atomic_csv import append_dict_rows_csv_atomic, write_dataframe_csv_atomic from shared.auction_model import predict_auction_price +from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path from shared.decision_economics import calculate_curve_decision_economics, derive_curve_verdict from shared.decision_policy import DecisionPolicyInput, derive_action_label, derive_action_label_from_row @@ -266,7 +267,7 @@ def load_cached_results() -> pd.DataFrame: return pd.DataFrame(columns=REQUIRED_COLUMNS) try: df = pd.read_csv(AI_RESULTS_PATH) - except Exception as exc: + except CSV_READ_ERRORS as exc: print(f"WARNING: could not read cached AI results {AI_RESULTS_PATH}: {type(exc).__name__}: {exc}") return pd.DataFrame(columns=REQUIRED_COLUMNS) missing = [column for column in REQUIRED_COLUMNS if column not in df.columns] @@ -732,7 +733,8 @@ def _dataset_contains_url(path: Path, url: str) -> bool: return False try: df = pd.read_csv(path, usecols=["url"], low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + print(f"WARNING: could not read {path} while checking for {url} ({type(exc).__name__}: {exc}).") return False if df.empty or "url" not in df.columns: return False @@ -1556,7 +1558,7 @@ def apply_platform_risk_adjustments( try: year = int(float(year_raw)) if year_raw not in (None, "") else None - except Exception: + except (TypeError, ValueError): year = None def add_flag(flag: str) -> None: diff --git a/scripts/ai_price_analysis.py b/scripts/ai_price_analysis.py index 0efb4ae0..bd766242 100644 --- a/scripts/ai_price_analysis.py +++ b/scripts/ai_price_analysis.py @@ -1,3 +1,4 @@ +import logging import re from dataclasses import dataclass from typing import Iterable, List, Optional @@ -7,8 +8,11 @@ from difflib import SequenceMatcher from pathlib import Path +from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path +logger = logging.getLogger(__name__) + ACTIVE_PRIMARY_PATH = dataset_path("active_vehicle_details.csv") ACTIVE_FALLBACK_PATH = dataset_path("vehicle_static_details.csv") @@ -184,7 +188,7 @@ def _infer_make_model_from_url(url_value: object) -> tuple[Optional[str], Option return None, None try: parsed = urlparse(str(url_value).strip()) - except Exception: + except ValueError: return None, None slug = parsed.path.rstrip("/").split("/")[-1] if not slug: @@ -322,9 +326,15 @@ def load_historical_sales( for source in extra_sources: try: normalised = _normalise_sold_dataframe(pd.read_csv(source)) - dataframes.append(normalised) - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Skipping unreadable sold history source %s (%s: %s); comps may be incomplete.", + source, + type(exc).__name__, + exc, + ) continue + dataframes.append(normalised) if not dataframes: return pd.DataFrame() @@ -872,7 +882,7 @@ def _prepare_match_rows( if "url" in df.columns: try: source_urls = df.loc[subset.index, "url"] - except Exception: # noqa: BLE001 + except KeyError: source_urls = df["url"].head(limit) def format_price(val): @@ -880,7 +890,7 @@ def format_price(val): return "—" try: return f"${float(val):,.0f}" - except Exception: + except (TypeError, ValueError): return str(val) def format_odometer(val): @@ -888,7 +898,7 @@ def format_odometer(val): return "—" try: return f"{int(round(float(val))):,} km" - except Exception: + except (TypeError, ValueError): text = str(val).strip() if not text: return "—" @@ -900,7 +910,7 @@ def format_year_with_delta(year_val, delta_val): try: base_year = int(round(float(year_val))) base_text = str(base_year) - except Exception: + except (TypeError, ValueError): base_text = str(year_val).strip() or "—" if not include_year_delta: return base_text @@ -908,7 +918,7 @@ def format_year_with_delta(year_val, delta_val): return f"{base_text} (+/-1y fallback)" try: delta_int = int(round(float(delta_val))) - except Exception: + except (TypeError, ValueError): return f"{base_text} (+/-1y fallback)" if delta_int == 0: return f"{base_text} (+/-1y fallback)" @@ -937,7 +947,7 @@ def format_reauction_count(value): return "" try: return int(round(float(value))) - except Exception: + except (TypeError, ValueError): return str(value) if "reauction_group_size" in subset.columns: diff --git a/scripts/extract_vehicle_details.py b/scripts/extract_vehicle_details.py index 07503b75..6d4955fb 100644 --- a/scripts/extract_vehicle_details.py +++ b/scripts/extract_vehicle_details.py @@ -18,6 +18,7 @@ sys.path.append(str(Path(__file__).resolve().parent.parent)) from scripts.atomic_csv import append_dataframe_csv_atomic, write_dataframe_csv_atomic + from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path from shared.schema import ACTIVE_DETAIL_SCHEMA, SOLD_RAW_SCRAPE_COLUMNS, STATIC_VEHICLE_SCHEMA from shared.sold_cleaning import ( @@ -256,7 +257,11 @@ def load_make_whitelist(existing_df: pd.DataFrame) -> set[str]: for make in sold_df["make"].dropna().unique().tolist() if str(make).strip() } - except Exception: + except CSV_READ_ERRORS as exc: + print( + f"WARNING: could not build make whitelist from {sold_path} " + f"({type(exc).__name__}: {exc}); falling back to existing listings." + ) whitelist = set() if not whitelist and "make" in existing_df.columns: whitelist = { @@ -723,10 +728,10 @@ def _parse_literal(raw: str) -> Any: normalized = _normalize_js_literals(raw) try: return ast.literal_eval(normalized) - except Exception: + except (ValueError, SyntaxError, MemoryError, RecursionError): try: return json.loads(raw) - except Exception: + except ValueError: return None diff --git a/scripts/process_curve_candidates.py b/scripts/process_curve_candidates.py index c49b4933..24a67793 100644 --- a/scripts/process_curve_candidates.py +++ b/scripts/process_curve_candidates.py @@ -22,12 +22,14 @@ from scripts.atomic_csv import write_dataframe_csv_atomic from scripts.curve_validator import build_curve_warnings from shared.canonical_tagging import tag_dataframe + from shared.csv_utils import CSV_READ_ERRORS from shared.curves import CURVE_COLUMNS, resolve_curve_canonical_tag from shared.data_loader import dataset_path else: # pragma: no cover from scripts.atomic_csv import write_dataframe_csv_atomic from scripts.curve_validator import build_curve_warnings from shared.canonical_tagging import tag_dataframe + from shared.csv_utils import CSV_READ_ERRORS from shared.curves import CURVE_COLUMNS, resolve_curve_canonical_tag from shared.data_loader import dataset_path @@ -222,7 +224,8 @@ def load_carsales_apify_market(path: Path | None = None) -> pd.DataFrame: return pd.DataFrame() try: df = pd.read_csv(csv_path, low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + print(f"WARNING: unreadable Autotrader market file {csv_path} ({type(exc).__name__}: {exc}).") return pd.DataFrame() if df.empty: return pd.DataFrame() diff --git a/scripts/run_grays_pipeline_loop.py b/scripts/run_grays_pipeline_loop.py index 593b546b..a4cb4589 100644 --- a/scripts/run_grays_pipeline_loop.py +++ b/scripts/run_grays_pipeline_loop.py @@ -13,8 +13,10 @@ if __package__ in (None, ""): sys.path.append(str(Path(__file__).resolve().parent.parent)) + from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path else: # pragma: no cover + from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path @@ -53,7 +55,8 @@ def _read_row_count(path: Path) -> int: return 0 try: df = pd.read_csv(path, low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + print(f"WARNING: could not count rows in {path} ({type(exc).__name__}: {exc}); treating as 0.") return 0 return len(df) diff --git a/scripts/scheduled_jobs.py b/scripts/scheduled_jobs.py index e5e31c10..3352d079 100644 --- a/scripts/scheduled_jobs.py +++ b/scripts/scheduled_jobs.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, Dict, Iterable, Optional import subprocess -from zoneinfo import ZoneInfo +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import pandas as pd import requests @@ -30,6 +30,7 @@ revalue_active_listings, ) from scripts.outcome_tracking import compute_outcome_metrics +from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path from shared.decision_policy import derive_action_label_from_row from shared.governance import write_governance_report_bundle @@ -183,7 +184,11 @@ def _local_timezone() -> timezone | ZoneInfo: timezone_name = os.getenv("AUTOSNIPER_LOCAL_TIMEZONE", "Australia/Sydney").strip() or "Australia/Sydney" try: return ZoneInfo(timezone_name) - except Exception: + except (ZoneInfoNotFoundError, ValueError, OSError) as exc: + print( + f"WARNING: unusable AUTOSNIPER_LOCAL_TIMEZONE {timezone_name!r} " + f"({type(exc).__name__}: {exc}); falling back to UTC coverage dates." + ) return timezone.utc @@ -233,7 +238,8 @@ def _load_daily_run_state() -> Dict[str, Any]: return {} try: return json.loads(DAILY_STATE_PATH.read_text(encoding="utf-8")) - except Exception: + except (OSError, ValueError) as exc: + print(f"WARNING: unreadable daily run state {DAILY_STATE_PATH} ({type(exc).__name__}: {exc}).") return {} @@ -283,7 +289,10 @@ def _last_successful_daily_date_local() -> date | None: def _read_lock_payload() -> Dict[str, Any] | None: try: return json.loads(LOCK_PATH.read_text(encoding="utf-8")) - except Exception: + except FileNotFoundError: + return None + except (OSError, ValueError) as exc: + print(f"WARNING: unreadable job lock {LOCK_PATH} ({type(exc).__name__}: {exc}).") return None @@ -392,7 +401,8 @@ def _load_existing_metrics() -> Dict[str, Any]: return {} try: return json.loads(METRICS_PATH.read_text(encoding="utf-8")) - except Exception: + except (OSError, ValueError) as exc: + print(f"WARNING: unreadable daily metrics {METRICS_PATH} ({type(exc).__name__}: {exc}).") return {} @@ -403,7 +413,8 @@ def _count_active_listings() -> Optional[int]: try: df = pd.read_csv(path, low_memory=False) return int(len(df)) - except Exception: + except CSV_READ_ERRORS as exc: + print(f"WARNING: could not count active listings in {path} ({type(exc).__name__}: {exc}).") return None @@ -481,7 +492,8 @@ def _load_daily_ai_analysis_frame() -> pd.DataFrame: if active_path.exists() and "url" in df.columns: try: active_df = pd.read_csv(active_path, usecols=["url"], low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + print(f"WARNING: unreadable active listings {active_path} ({type(exc).__name__}: {exc}).") active_df = pd.DataFrame() if not active_df.empty and "url" in active_df.columns: active_urls = set(active_df["url"].dropna().astype(str).str.strip()) @@ -530,8 +542,8 @@ def _send_daily_ai_analysis_summary(*, trigger: str, coverage_date_local: date) ) print(f"Daily AI Analysis Telegram summary sent={sent}, verdict={verdict}.") return sent - except Exception: - print("Daily AI Analysis Telegram summary failed.") + except Exception as exc: # noqa: BLE001 - summary alert must not fail the pipeline + print(f"WARNING: Daily AI Analysis Telegram summary failed: {type(exc).__name__}: {exc}") return False @@ -556,9 +568,8 @@ def _wait_for_internet(max_wait_hours: int) -> bool: def _existing_lock_ttl_hours() -> int: - try: - payload = json.loads(LOCK_PATH.read_text(encoding="utf-8")) - except Exception: + payload = _read_lock_payload() + if not payload: return max(max(LOCK_TTLS.values(), default=4), 4) owner_job = str(payload.get("job") or "").strip() return LOCK_TTLS.get(owner_job, max(max(LOCK_TTLS.values(), default=4), 4)) @@ -846,7 +857,8 @@ def _load_external_auction_seed_listings(output_dir: Path) -> list[scrape_extern continue try: df = pd.read_csv(path, low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + print(f"WARNING: skipping unreadable external auction seed {path} ({type(exc).__name__}: {exc}).") continue if df.empty or "url" not in df.columns: continue diff --git a/scripts/scrape_autotrader_rego.py b/scripts/scrape_autotrader_rego.py index a5d436a1..2a9fa2d6 100644 --- a/scripts/scrape_autotrader_rego.py +++ b/scripts/scrape_autotrader_rego.py @@ -99,8 +99,11 @@ def main() -> None: done = set(progress.get("completed", [])) urls = [url for url in urls if url not in done] print(f"Resuming: {len(done)} already completed, {len(urls)} remaining.") - except Exception: - pass + except (OSError, ValueError, AttributeError) as exc: + print( + f"WARNING: could not read resume progress {PROGRESS_PATH} " + f"({type(exc).__name__}: {exc}); processing every URL." + ) if args.limit and args.limit > 0: urls = urls[: args.limit] if args.max_per_run and args.max_per_run > 0: diff --git a/scripts/scrape_bid_history.py b/scripts/scrape_bid_history.py index dd95ca7f..e9e39a22 100644 --- a/scripts/scrape_bid_history.py +++ b/scripts/scrape_bid_history.py @@ -17,9 +17,11 @@ sys.path.append(str(Path(__file__).resolve().parent.parent)) from scripts.atomic_csv import append_dict_rows_csv_atomic + from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path else: # pragma: no cover from scripts.atomic_csv import append_dict_rows_csv_atomic + from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path @@ -58,8 +60,11 @@ def _load_urls( existing = pd.read_csv(output_path, usecols=["url"]) existing_urls = set(existing["url"].dropna().astype(str).str.strip()) urls = urls[~urls.isin(existing_urls)] - except Exception: - pass + except CSV_READ_ERRORS as exc: + print( + f"WARNING: could not read existing bid history {output_path} " + f"({type(exc).__name__}: {exc}); re-scraping every URL." + ) if limit is not None and limit > 0: urls = urls.head(limit) return urls.tolist() diff --git a/scripts/update_bids.py b/scripts/update_bids.py index d9b93a27..54a5c23e 100644 --- a/scripts/update_bids.py +++ b/scripts/update_bids.py @@ -19,12 +19,14 @@ sys.path.append(str(Path(__file__).resolve().parent.parent)) from scripts.active_snapshot_retention import compact_active_snapshots from scripts.atomic_csv import append_dict_rows_csv_atomic, write_dataframe_csv_atomic + from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path from shared.schema import STATE_ACTIVE, STATE_STATIC_PARSED from shared.state_machine import ListingObservation, ensure_state_schema, upsert_state_row else: # pragma: no cover from scripts.active_snapshot_retention import compact_active_snapshots from scripts.atomic_csv import append_dict_rows_csv_atomic, write_dataframe_csv_atomic + from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path from shared.schema import STATE_ACTIVE, STATE_STATIC_PARSED from shared.state_machine import ListingObservation, ensure_state_schema, upsert_state_row @@ -129,11 +131,8 @@ def derive_location_state(location: str | None) -> str: def derive_auction_site(url: str | None) -> str: if not url: return "" - try: - parts = url.split("/") - return parts[3] if len(parts) > 3 else "" - except Exception: - return "" + parts = str(url).split("/") + return parts[3] if len(parts) > 3 else "" def append_snapshot_row(record: dict[str, object]) -> None: @@ -147,14 +146,12 @@ def record_snapshot( bids_text: str | None, time_remaining_text: str | None, ) -> None: - try: - row = df.loc[df["url"] == url].iloc[0].to_dict() - except Exception: - row = {} + matches = df.loc[df["url"] == url] + row = matches.iloc[0].to_dict() if not matches.empty else {} price_numeric = parse_currency_value(price_text) try: bids_numeric = int(bids_text) - except Exception: + except (TypeError, ValueError): bids_numeric = None time_hours = parse_time_remaining_to_hours(time_remaining_text) record = { @@ -181,7 +178,13 @@ def load_resume_queue(all_urls: list[str]) -> list[str]: try: data = json.loads(resume_path.read_text(encoding="utf-8")) queued = data.get("remaining_urls", []) - except Exception: + except (OSError, ValueError, AttributeError) as exc: + logger.warning( + "Ignoring unreadable resume queue %s (%s: %s); reprocessing the full URL list.", + resume_path, + type(exc).__name__, + exc, + ) return all_urls if not queued: return all_urls @@ -229,7 +232,13 @@ def _load_state_dataframe() -> pd.DataFrame: return ensure_state_schema(pd.DataFrame()) try: df = pd.read_csv(STATE_FILE, low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + logger.error( + "Unreadable listing state %s (%s: %s); rebuilding state from an empty frame.", + STATE_FILE, + type(exc).__name__, + exc, + ) df = pd.DataFrame() return ensure_state_schema(df) @@ -284,7 +293,13 @@ def _load_active_queue_urls() -> set[str]: return set() try: links_df = pd.read_csv(ACTIVE_LINKS_FILE) - except Exception: + except CSV_READ_ERRORS as exc: + logger.error( + "Unreadable active link queue %s (%s: %s); skipping state reconciliation.", + ACTIVE_LINKS_FILE, + type(exc).__name__, + exc, + ) return set() if "url" not in links_df.columns: return set() @@ -603,7 +618,7 @@ async def update_bids( df = _load_active_seed_dataframe() if df.empty: print("No active seed dataset found. Expected active or static listings CSV.") - return [], skipped_urls + return df, skipped_urls state_df = _load_state_dataframe() state_df, reconciled_rows = reconcile_state_active_queue(state_df) if reconciled_rows: @@ -855,16 +870,13 @@ async def update_bids( clear_resume_queue() if not skip_master: - try: - from scripts import update_master + from scripts import update_master - update_master.update_master_database() - except Exception as exc: - logger.error(f"Failed to run update_master after update_bids: {exc}") + update_master.update_master_database() return df, skipped_urls - except Exception as e: - logger.error(f"Unexpected error in update_bids: {e}") + except Exception: + logger.exception("Unexpected error in update_bids; attempting emergency snapshot before failing.") if "df" in locals(): try: persist_dataframe(df, "Emergency save") @@ -872,7 +884,7 @@ async def update_bids( persist_state_dataframe(state_df, "Emergency state save") except Exception as save_error: # noqa: BLE001 logger.error(f"Failed to persist emergency snapshot: {save_error}") - return df, skipped_urls + raise # ─── Entry point ──────────────────────────────────────────────── if __name__ == "__main__": diff --git a/scripts/update_master.py b/scripts/update_master.py index d623d9a5..12051165 100644 --- a/scripts/update_master.py +++ b/scripts/update_master.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import logging import re import sys from datetime import datetime, timezone @@ -15,6 +16,7 @@ if __package__ in (None, ""): sys.path.append(str(Path(__file__).resolve().parent.parent)) from scripts.atomic_csv import append_dataframe_csv_atomic, write_dataframe_csv_atomic + from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path from shared.governance import SOLD_DETAIL_SCHEMA from shared.sold_cleaning import normalize_listing_fields @@ -26,6 +28,7 @@ from scripts.build_restricted_datasets import build_restricted_datasets else: from scripts.atomic_csv import append_dataframe_csv_atomic, write_dataframe_csv_atomic + from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path from shared.governance import SOLD_DETAIL_SCHEMA from shared.sold_cleaning import normalize_listing_fields @@ -35,6 +38,9 @@ from shared.validators import R, validate_sold_cars_df from shared.exclusions import append_pipeline_exclusions from scripts.build_restricted_datasets import build_restricted_datasets + +logger = logging.getLogger(__name__) + SOLD_FILE = dataset_path("sold_cars.csv") REFERRED_FILE = dataset_path("referred_cars.csv") ACTIVE_FILE = dataset_path("active_vehicle_details.csv") @@ -403,7 +409,7 @@ def _merge_preserving_history( if prepare_fn is not None: try: schema_changed = not prepared_existing.equals(existing_raw) - except Exception: + except (TypeError, ValueError): schema_changed = True if prepared_new.empty: @@ -658,7 +664,13 @@ def _load_state_table() -> pd.DataFrame: return ensure_state_schema(pd.DataFrame()) try: state_df = pd.read_csv(STATE_FILE, low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + logger.error( + "Unreadable listing state %s (%s: %s); rebuilding state from an empty frame.", + STATE_FILE, + type(exc).__name__, + exc, + ) state_df = pd.DataFrame() return ensure_state_schema(state_df) @@ -878,7 +890,7 @@ def update_master_database() -> None: try: build_restricted_datasets() except Exception as exc: - print(f"Restricted dataset build failed: {exc}") + raise RuntimeError("Restricted dataset build failed after master update.") from exc if __name__ == "__main__": diff --git a/shared/auction_model.py b/shared/auction_model.py index 1c721b9d..d21d8cd5 100644 --- a/shared/auction_model.py +++ b/shared/auction_model.py @@ -16,6 +16,7 @@ from __future__ import annotations import json +import logging from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping, Optional @@ -23,6 +24,8 @@ import numpy as np import pandas as pd +logger = logging.getLogger(__name__) + _ARTIFACTS = Path(__file__).resolve().parent.parent / "artifacts" _MODEL_Q50_PATH = _ARTIFACTS / "auction_ratio_q50.cbm" _MODEL_Q90_PATH = _ARTIFACTS / "auction_ratio_q90.cbm" @@ -76,7 +79,12 @@ def _try_load_models() -> bool: _models_available = True return True - except Exception: + except Exception as exc: # noqa: BLE001 - model load failures degrade to curve-only pricing + logger.error( + "Auction model load failed (%s: %s); valuations fall back to curve-only pricing.", + type(exc).__name__, + exc, + ) _models_available = False return False @@ -275,7 +283,12 @@ def predict_auction_price( "calibration_multiplier": _calibration_multiplier, "comps_p50": comps_p50, } - except Exception: + except Exception as exc: # noqa: BLE001 - prediction failures degrade to curve-only pricing + logger.error( + "Auction model prediction failed (%s: %s); returning no model prediction.", + type(exc).__name__, + exc, + ) return None diff --git a/shared/canonical_tagging.py b/shared/canonical_tagging.py index 299e1003..6ad9eb3d 100644 --- a/shared/canonical_tagging.py +++ b/shared/canonical_tagging.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from datetime import datetime, timezone import json +import logging import re from functools import lru_cache from pathlib import Path @@ -12,12 +13,15 @@ import pandas as pd +from shared.csv_utils import CSV_READ_ERRORS from shared.curve_groups_v2 import load_curve_anchor_overrides_v2, load_curve_groups_v2 from shared.curves import resolve_curve_canonical_tag from shared.data_loader import dataset_path from shared.validators import R from scripts.atomic_csv import write_dataframe_csv_atomic +logger = logging.getLogger(__name__) + UNCLASSIFIED = "UNCLASSIFIED" AMBIG_BADGE = "[AMBIG_BADGE]" @@ -509,7 +513,13 @@ def _load_curve_year_band() -> pd.DataFrame | None: if curve_path.exists(): try: curves_df = pd.read_csv(curve_path) - except Exception: + except CSV_READ_ERRORS as exc: + logger.error( + "Unreadable curve file %s (%s: %s); year bands will be missing.", + curve_path, + type(exc).__name__, + exc, + ) curves_df = pd.DataFrame() if "canonical_tag" in curves_df.columns and "anchor_year" in curves_df.columns: curve_band = ( @@ -527,7 +537,12 @@ def _load_curve_year_band() -> pd.DataFrame | None: try: overrides_df = load_curve_anchor_overrides_v2() groups_df = load_curve_groups_v2() - except Exception: + except (OSError, ValueError) as exc: + logger.error( + "Could not load curve group config (%s: %s); anchor overrides will be ignored.", + type(exc).__name__, + exc, + ) overrides_df = pd.DataFrame() groups_df = pd.DataFrame() diff --git a/shared/csv_utils.py b/shared/csv_utils.py index 28f08f91..419bad7c 100644 --- a/shared/csv_utils.py +++ b/shared/csv_utils.py @@ -3,11 +3,18 @@ from __future__ import annotations import csv +import logging from pathlib import Path from typing import Any import pandas as pd +logger = logging.getLogger(__name__) + +# Failures expected when a CSV exists but cannot be parsed as tabular data. +# pandas parser errors (ParserError, EmptyDataError) all derive from ValueError. +CSV_READ_ERRORS: tuple[type[BaseException], ...] = (OSError, ValueError, csv.Error) + def count_csv_records(path: Path | str) -> int | None: """Count logical CSV records, including rows with quoted embedded newlines.""" @@ -17,7 +24,8 @@ def count_csv_records(path: Path | str) -> int | None: try: with file_path.open("r", encoding="utf-8", errors="ignore", newline="") as handle: return max(sum(1 for _ in csv.reader(handle)) - 1, 0) - except (OSError, csv.Error): + except (OSError, csv.Error) as exc: + logger.warning("Could not count records in %s (%s: %s).", file_path, type(exc).__name__, exc) return None @@ -35,5 +43,11 @@ def read_csv_or_empty(path: Path | str, **kwargs: Any) -> pd.DataFrame: return pd.DataFrame() try: return read_csv_stable(file_path, **kwargs) - except (FileNotFoundError, ValueError, pd.errors.EmptyDataError): + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable CSV %s (%s: %s); continuing with an empty frame.", + file_path, + type(exc).__name__, + exc, + ) return pd.DataFrame() diff --git a/shared/curves.py b/shared/curves.py index a30f92f9..575c6861 100644 --- a/shared/curves.py +++ b/shared/curves.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from functools import lru_cache from pathlib import Path from typing import Iterable, List, Mapping, Optional, Sequence, Tuple @@ -9,10 +10,13 @@ import pandas as pd from shared.audit import append_audit_snapshot +from shared.csv_utils import CSV_READ_ERRORS from shared.curve_groups_v2 import load_curve_groups_v2, resolve_base_curve_tag from shared.curve_versioning import snapshot_curve_version from shared.data_loader import dataset_path +logger = logging.getLogger(__name__) + CURVE_COLUMNS: Sequence[str] = ( "canonical_tag", @@ -111,7 +115,13 @@ def load_saved_curve_tags(path: Path | None = None) -> set[str]: return set() try: df = pd.read_csv(curve_path, usecols=["canonical_tag"]) - except Exception: + except CSV_READ_ERRORS as exc: + logger.error( + "Unreadable curve file %s (%s: %s); treating the curve universe as empty.", + curve_path, + type(exc).__name__, + exc, + ) return set() return { str(value).strip() diff --git a/shared/data_loader.py b/shared/data_loader.py index 614e09fb..e166b887 100644 --- a/shared/data_loader.py +++ b/shared/data_loader.py @@ -28,6 +28,7 @@ import io import json +import logging import os import time import zipfile @@ -36,6 +37,8 @@ import requests +logger = logging.getLogger(__name__) + DATA_DIR = Path(os.getenv("AUTOSNIPER_DATA_DIR", "CSV_data")) DATASET_PATHS: dict[str, Path] = { @@ -118,13 +121,19 @@ def _should_refresh(cache_minutes: int) -> bool: return True try: info = json.loads(_SYNC_MARKER.read_text(encoding="utf-8")) - except Exception: + timestamp = float(info.get("timestamp", 0)) + except (OSError, ValueError, TypeError, AttributeError) as exc: + logger.warning( + "Unreadable data sync marker %s (%s: %s); forcing a refresh.", + _SYNC_MARKER, + type(exc).__name__, + exc, + ) return True - timestamp = info.get("timestamp", 0) url = info.get("url") if url != os.getenv("AUTOSNIPER_DATA_URL"): return True - return (time.time() - float(timestamp)) > cache_minutes * 60 + return (time.time() - timestamp) > cache_minutes * 60 def _extract_zip(content: bytes) -> None: @@ -207,8 +216,13 @@ def upload_remote_data_bundle(filenames: Iterable[str] | None = None) -> bool: response = requests.put(upload_url, headers=headers, data=payload, timeout=timeout) response.raise_for_status() return True - except Exception: - # Avoid crashing the UI if upload fails. + except Exception as exc: # noqa: BLE001 - remote publication must not crash the UI + # Avoid crashing the UI if upload fails, but never fail silently. + logger.error( + "Remote data bundle upload failed (%s: %s); local CSVs were not published.", + type(exc).__name__, + exc, + ) return False diff --git a/shared/manual_curve_evidence.py b/shared/manual_curve_evidence.py index 9ef961f1..eed83ed2 100644 --- a/shared/manual_curve_evidence.py +++ b/shared/manual_curve_evidence.py @@ -1,12 +1,16 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Sequence import pandas as pd +from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path +logger = logging.getLogger(__name__) + MANUAL_CURVE_EVIDENCE_PATH = dataset_path("quality/manual_curve_evidence.csv") MANUAL_CURVE_EVIDENCE_COLUMNS: Sequence[str] = ( @@ -31,7 +35,13 @@ def load_manual_curve_evidence(path: Path | None = None) -> pd.DataFrame: return pd.DataFrame(columns=list(MANUAL_CURVE_EVIDENCE_COLUMNS)) try: df = pd.read_csv(csv_path, low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable manual curve evidence %s (%s: %s); continuing without it.", + csv_path, + type(exc).__name__, + exc, + ) return pd.DataFrame(columns=list(MANUAL_CURVE_EVIDENCE_COLUMNS)) for column in MANUAL_CURVE_EVIDENCE_COLUMNS: if column not in df.columns: diff --git a/shared/missed_opportunities.py b/shared/missed_opportunities.py index 5392a646..947880c3 100644 --- a/shared/missed_opportunities.py +++ b/shared/missed_opportunities.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any, Mapping +import logging import os import re @@ -28,12 +29,15 @@ apply_platform_risk_adjustments, ) from shared.comps_engine import parse_currency, parse_numeric +from shared.csv_utils import CSV_READ_ERRORS from shared.decision_economics import calculate_curve_decision_economics, derive_curve_verdict from shared.decision_policy import derive_action_label_from_row from shared.curves import resolve_curve_canonical_tag from shared.repair_pricing import assess_repairs, repair_decision_label, vehicle_class_for_listing from shared.sold_comparables import select_km_aware_comparables +logger = logging.getLogger(__name__) + COMPS_STATS_COLUMNS = ["comps_count", "comps_median", "comps_mean", "comps_min", "comps_max"] COMPS_STATS_INTERNAL_COLUMNS = ["comps_prices", "comps_urls", "comps_odometers"] EXTERNAL_AUCTION_MATCHES_FILENAME = "external_auction_curve_matches.csv" @@ -92,7 +96,13 @@ def load_external_auction_sold_rows(path: Path | None = None) -> pd.DataFrame: return pd.DataFrame() try: df = pd.read_csv(source_path, low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable external auction matches %s (%s: %s); missed opportunities will be incomplete.", + source_path, + type(exc).__name__, + exc, + ) return pd.DataFrame() if df.empty or "url" not in df.columns: return pd.DataFrame() diff --git a/shared/project_memory.py b/shared/project_memory.py index f8804481..93b5bafa 100644 --- a/shared/project_memory.py +++ b/shared/project_memory.py @@ -221,7 +221,7 @@ def _extract_pipeline_stage_choices(root: Path = REPO_ROOT) -> list[str]: continue try: value = ast.literal_eval(keyword.value) - except Exception: + except ValueError: continue if isinstance(value, (list, tuple)) and all(isinstance(item, str) for item in value): discovered = list(value) diff --git a/shared/repair_ai_classifier.py b/shared/repair_ai_classifier.py index 13e6aade..ff43c897 100644 --- a/shared/repair_ai_classifier.py +++ b/shared/repair_ai_classifier.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import logging import os from dataclasses import dataclass from datetime import datetime, timezone @@ -10,8 +11,11 @@ import pandas as pd import yaml +from shared.csv_utils import CSV_READ_ERRORS from shared.repair_review import LIVE_QUEUE_PATH, safe_text +logger = logging.getLogger(__name__) + REPORT_DIR = Path("CSV_data/reports") AI_SUGGESTIONS_PATH = REPORT_DIR / "repair_review_ai_suggestions.csv" @@ -71,7 +75,13 @@ def load_ai_suggestions(path: Path = AI_SUGGESTIONS_PATH) -> pd.DataFrame: return pd.DataFrame(columns=AI_SUGGESTION_COLUMNS) try: df = pd.read_csv(path).fillna("") - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable repair AI suggestions %s (%s: %s); previous suggestions will be ignored.", + path, + type(exc).__name__, + exc, + ) return pd.DataFrame(columns=AI_SUGGESTION_COLUMNS) for column in AI_SUGGESTION_COLUMNS: if column not in df.columns: @@ -104,7 +114,13 @@ def _dictionary_vocab(path: Path = DICTIONARY_PATH) -> dict[str, list[str]]: return {"categories": CATEGORY_OPTIONS, "canonical_defects": []} try: payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - except Exception: + except (OSError, yaml.YAMLError) as exc: + logger.warning( + "Unreadable condition dictionary %s (%s: %s); falling back to default vocabulary.", + path, + type(exc).__name__, + exc, + ) return {"categories": CATEGORY_OPTIONS, "canonical_defects": []} entries = payload.get("entries") or [] defects = sorted( @@ -299,7 +315,8 @@ def classify_repair_review_queue( model_name = model or os.getenv("AUTOSNIPER_REPAIR_AI_MODEL", "gpt-4.1-mini") try: new_suggestions = caller(pending, model=model_name) if caller is not None else _call_openai(pending, model=model_name) - except Exception as exc: + except Exception as exc: # noqa: BLE001 - reported to the caller via skipped_reason + logger.error("Repair AI classification call failed (%s: %s).", type(exc).__name__, exc) return ClassifierResult( len(pending), 0, diff --git a/shared/repair_pricing.py b/shared/repair_pricing.py index a21a489c..c7dcc4fe 100644 --- a/shared/repair_pricing.py +++ b/shared/repair_pricing.py @@ -9,15 +9,19 @@ from typing import Dict, List, Mapping, Optional, Tuple import html import json +import logging import re import pandas as pd import yaml +from shared.csv_utils import CSV_READ_ERRORS from shared.condition_normalizer import estimate_component_count from shared.repair_features import build_repair_features from shared.repair_review import DECISIONS_PATH, load_repair_review_decisions, review_key, safe_text +logger = logging.getLogger(__name__) + PANEL_RATE = 300 PANEL_CAP = 3 @@ -148,7 +152,13 @@ def _load_schedule_cost_bands(_path: str, _content_hash: str) -> Dict[tuple[str, """Load class-specific low/default/high estimates for one file signature.""" try: df = pd.read_csv(REPAIR_PRICING_SCHEDULE_PATH) - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable repair pricing schedule %s (%s: %s); using default cost bands.", + REPAIR_PRICING_SCHEDULE_PATH, + type(exc).__name__, + exc, + ) return {} overrides: Dict[tuple[str, str], RepairCostBand] = {} for _, row in df.iterrows(): diff --git a/shared/repair_pricing_schedule.py b/shared/repair_pricing_schedule.py index 7ee2cbe1..0a2bb240 100644 --- a/shared/repair_pricing_schedule.py +++ b/shared/repair_pricing_schedule.py @@ -1,14 +1,18 @@ from __future__ import annotations from datetime import date +import logging import re from pathlib import Path import pandas as pd import yaml +from shared.csv_utils import CSV_READ_ERRORS from shared.repair_review import DECISIONS_PATH, load_repair_review_decisions, safe_text +logger = logging.getLogger(__name__) + REPORT_DIR = Path("CSV_data/reports") DICTIONARY_PATH = Path("config/condition_dictionary_v2.yaml") @@ -234,7 +238,13 @@ def load_pricing_schedule(path: Path = PRICING_SCHEDULE_PATH) -> pd.DataFrame: return _blank_pricing_frame() try: df = pd.read_csv(path).fillna("") - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable pricing schedule %s (%s: %s); starting from a blank schedule.", + path, + type(exc).__name__, + exc, + ) return _blank_pricing_frame() for column in PRICING_COLUMNS: if column not in df.columns: @@ -299,7 +309,13 @@ def load_quote_requests(path: Path = QUOTE_REQUESTS_PATH) -> pd.DataFrame: return _blank_quote_frame() try: df = pd.read_csv(path).fillna("") - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable quote requests %s (%s: %s); starting from a blank quote list.", + path, + type(exc).__name__, + exc, + ) return _blank_quote_frame() for column in QUOTE_COLUMNS: if column not in df.columns: @@ -528,7 +544,13 @@ def dictionary_pricing_candidates(path: Path = DICTIONARY_PATH) -> pd.DataFrame: return pd.DataFrame(columns=["canonical_defect", "category", "examples", "decision_count"]) try: payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - except Exception: + except (OSError, yaml.YAMLError) as exc: + logger.warning( + "Unreadable condition dictionary %s (%s: %s); no pricing candidates derived.", + path, + type(exc).__name__, + exc, + ) return pd.DataFrame(columns=["canonical_defect", "category", "examples", "decision_count"]) rows: list[dict[str, object]] = [] diff --git a/shared/repair_review.py b/shared/repair_review.py index ca2ef5cb..f44e77a8 100644 --- a/shared/repair_review.py +++ b/shared/repair_review.py @@ -2,10 +2,15 @@ from pathlib import Path from typing import Iterable +import logging import re import pandas as pd +from shared.csv_utils import CSV_READ_ERRORS + +logger = logging.getLogger(__name__) + REPORT_DIR = Path("CSV_data/reports") DECISIONS_PATH = REPORT_DIR / "repair_review_decisions.csv" @@ -74,7 +79,13 @@ def load_repair_review_decisions(path: Path = DECISIONS_PATH) -> pd.DataFrame: return pd.DataFrame(columns=REVIEW_COLUMNS) try: df = pd.read_csv(path).fillna("") - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable repair review decisions %s (%s: %s); treating them as empty.", + path, + type(exc).__name__, + exc, + ) return pd.DataFrame(columns=REVIEW_COLUMNS) for column in REVIEW_COLUMNS: if column not in df.columns: diff --git a/shared/scraper_health.py b/shared/scraper_health.py index a6fc8744..2ef54f2e 100644 --- a/shared/scraper_health.py +++ b/shared/scraper_health.py @@ -3,14 +3,18 @@ from __future__ import annotations import json +import logging from datetime import datetime, timezone from pathlib import Path from typing import Any import pandas as pd +from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path +logger = logging.getLogger(__name__) + ROOT_DIR = Path(__file__).resolve().parent.parent DEFAULT_HEALTH_REPORT_DIR = ROOT_DIR / "output" / "health" @@ -79,7 +83,13 @@ def _load_csv(path: Path) -> pd.DataFrame: return pd.DataFrame() try: return pd.read_csv(path, low_memory=False) - except (ValueError, pd.errors.EmptyDataError): + except CSV_READ_ERRORS as exc: + logger.warning( + "Health check could not read %s (%s: %s); reporting it as empty.", + path, + type(exc).__name__, + exc, + ) return pd.DataFrame() @@ -116,7 +126,13 @@ def _top_failure_reasons() -> pd.DataFrame: return pd.DataFrame(columns=["reason_code", "count"]) try: df = pd.read_csv(path, usecols=["reason_code"], low_memory=False) - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Could not read scraper failure reasons from %s (%s: %s).", + path, + type(exc).__name__, + exc, + ) return pd.DataFrame(columns=["reason_code", "count"]) if df.empty or "reason_code" not in df.columns: return pd.DataFrame(columns=["reason_code", "count"]) @@ -255,5 +271,11 @@ def load_scraper_health_report(report_path: Path = SCRAPER_HEALTH_JSON_PATH) -> return None try: return json.loads(report_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): + except (json.JSONDecodeError, OSError) as exc: + logger.warning( + "Unreadable scraper health report %s (%s: %s).", + report_path, + type(exc).__name__, + exc, + ) return None diff --git a/shared/telegram_alerts.py b/shared/telegram_alerts.py index ff0527ae..d7030648 100644 --- a/shared/telegram_alerts.py +++ b/shared/telegram_alerts.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os from datetime import datetime, timezone from typing import Optional @@ -7,8 +8,11 @@ import pandas as pd import requests +from shared.csv_utils import CSV_READ_ERRORS from shared.data_loader import dataset_path +logger = logging.getLogger(__name__) + ALERT_LOG_PATH = dataset_path("ai/telegram_alert_log.csv") ALERT_LOG_COLUMNS = [ @@ -53,7 +57,13 @@ def _load_alert_log() -> pd.DataFrame: return pd.DataFrame(columns=ALERT_LOG_COLUMNS) try: df = pd.read_csv(ALERT_LOG_PATH) - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable Telegram alert log %s (%s: %s); duplicate alerts may be re-sent.", + ALERT_LOG_PATH, + type(exc).__name__, + exc, + ) return pd.DataFrame(columns=ALERT_LOG_COLUMNS) for column in ALERT_LOG_COLUMNS: if column not in df.columns: @@ -83,7 +93,13 @@ def _load_alert_state() -> pd.DataFrame: return pd.DataFrame(columns=ALERT_STATE_COLUMNS) try: df = pd.read_csv(ALERT_STATE_PATH) - except Exception: + except CSV_READ_ERRORS as exc: + logger.warning( + "Unreadable Telegram alert state %s (%s: %s); state-change alerts may be re-sent.", + ALERT_STATE_PATH, + type(exc).__name__, + exc, + ) return pd.DataFrame(columns=ALERT_STATE_COLUMNS) for column in ALERT_STATE_COLUMNS: if column not in df.columns: diff --git a/shared/ui_helpers.py b/shared/ui_helpers.py index 314c1e6d..028e3b4f 100644 --- a/shared/ui_helpers.py +++ b/shared/ui_helpers.py @@ -13,7 +13,8 @@ def display_profit_bar(profit_str: str, verdict: str) -> None: """ try: percent = float(str(profit_str).strip('%')) - st.markdown(f"**Profit Margin: {profit_str} — Verdict: {verdict}**") - st.progress(min(percent / 100, 1.0)) - except Exception: - st.warning("⚠️ Could not parse profit margin.") + except (TypeError, ValueError): + st.warning(f"⚠️ Could not parse profit margin from {profit_str!r}.") + return + st.markdown(f"**Profit Margin: {profit_str} — Verdict: {verdict}**") + st.progress(min(percent / 100, 1.0)) diff --git a/status_app.py b/status_app.py index d6bddaac..c55aa287 100644 --- a/status_app.py +++ b/status_app.py @@ -16,15 +16,16 @@ def load_metrics() -> dict: return {} try: return json.loads(METRICS_PATH.read_text(encoding="utf-8")) - except Exception: + except (OSError, ValueError) as exc: + st.warning(f"Unreadable pipeline metrics {METRICS_PATH}: {type(exc).__name__}: {exc}") return {} def format_minutes_ago(last_run_utc: str) -> float: try: - ts = datetime.fromisoformat(last_run_utc.replace("Z", "+00:00")) + ts = datetime.fromisoformat(str(last_run_utc).replace("Z", "+00:00")) return max((NOW - ts).total_seconds() / 60.0, 0.0) - except Exception: + except (TypeError, ValueError): return math.inf diff --git a/tests/test_error_visibility.py b/tests/test_error_visibility.py new file mode 100644 index 00000000..6dc5e79d --- /dev/null +++ b/tests/test_error_visibility.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import logging +from pathlib import Path + +import pandas as pd +import pytest + +import scripts.update_master as update_master +import shared.csv_utils as csv_utils +import shared.scraper_health as scraper_health +import shared.telegram_alerts as telegram_alerts + +MALFORMED_CSV = 'url,value\n"unterminated,1\n' + + +def _write_malformed(path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(MALFORMED_CSV, encoding="utf-8") + return path + + +def test_read_csv_or_empty_logs_unreadable_file(tmp_path, caplog): + path = _write_malformed(tmp_path / "broken.csv") + + with caplog.at_level(logging.WARNING, logger="shared.csv_utils"): + df = csv_utils.read_csv_or_empty(path) + + assert df.empty + assert "Unreadable CSV" in caplog.text + assert "broken.csv" in caplog.text + + +def test_count_csv_records_logs_unreadable_file(tmp_path, caplog, monkeypatch): + path = tmp_path / "counts.csv" + path.write_text("url\na\n", encoding="utf-8") + + def _boom(*_args, **_kwargs): + raise OSError("disk gone") + + monkeypatch.setattr(Path, "open", _boom) + + with caplog.at_level(logging.WARNING, logger="shared.csv_utils"): + assert csv_utils.count_csv_records(path) is None + + assert "Could not count records" in caplog.text + + +def test_load_alert_log_logs_unreadable_file(tmp_path, caplog, monkeypatch): + path = _write_malformed(tmp_path / "telegram_alert_log.csv") + monkeypatch.setattr(telegram_alerts, "ALERT_LOG_PATH", path) + + with caplog.at_level(logging.WARNING, logger="shared.telegram_alerts"): + df = telegram_alerts._load_alert_log() + + assert list(df.columns) == list(telegram_alerts.ALERT_LOG_COLUMNS) + assert "duplicate alerts may be re-sent" in caplog.text + + +def test_scraper_health_load_csv_logs_unreadable_file(tmp_path, caplog): + path = _write_malformed(tmp_path / "health.csv") + + with caplog.at_level(logging.WARNING, logger="shared.scraper_health"): + df = scraper_health._load_csv(path) + + assert df.empty + assert "could not read" in caplog.text + + +def test_load_state_table_logs_unreadable_state_file(tmp_path, caplog, monkeypatch): + path = _write_malformed(tmp_path / "vehicle_state.csv") + monkeypatch.setattr(update_master, "STATE_FILE", path) + + with caplog.at_level(logging.ERROR, logger="scripts.update_master"): + state_df = update_master._load_state_table() + + assert isinstance(state_df, pd.DataFrame) + assert "Unreadable listing state" in caplog.text + + +def test_update_master_propagates_restricted_build_failure(tmp_path, monkeypatch): + def _tmp_dataset_path(name: str) -> Path: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + return path + + for attribute in ( + "SOLD_FILE", + "REFERRED_FILE", + "ACTIVE_FILE", + "STATIC_FILE", + "STATE_FILE", + "SOLD_DISCARD_LOG", + "NORMALIZED_FILE", + ): + monkeypatch.setattr(update_master, attribute, _tmp_dataset_path(attribute.lower() + ".csv")) + monkeypatch.setattr(update_master, "dataset_path", _tmp_dataset_path) + monkeypatch.setattr(update_master, "_load_dataframe", lambda *_a, **_k: pd.DataFrame()) + monkeypatch.setattr(update_master, "append_pipeline_exclusions", lambda *_a, **_k: None) + monkeypatch.setattr( + update_master, + "_load_state_table", + lambda: update_master.ensure_state_schema( + pd.DataFrame([{"url": "https://example.com/lot/1", "state": "active"}]) + ), + ) + + def _boom() -> None: + raise ValueError("restricted build exploded") + + monkeypatch.setattr(update_master, "build_restricted_datasets", _boom) + + with pytest.raises(RuntimeError, match="Restricted dataset build failed"): + update_master.update_master_database()