Surface swallowed errors across pipeline and UI

ops/active_monitor.py

@@ -3,6 +3,7 @@
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Iterable
+import logging
 import os
 import re
 
@@ -20,12 +21,14 @@
 from scripts.process_curve_candidates import DEFAULT_AUTOTRADER_SOURCE, load_autotrader_market
 from shared.canonical_tagging import UNCLASSIFIED, is_canonical_eligible
 from shared.comps_engine import parse_currency, parse_numeric
+from shared.csv_utils import CSV_READ_ERRORS
 from shared.curves import interpolate_base_by_year, list_curve_tags, load_curves, resolve_curve_canonical_tag
 from shared.data_loader import dataset_path
 from shared.location_utils import extract_state
 from shared.repair_pricing import assess_repairs, repair_fragments_to_records
 from shared.repair_review import append_live_review_items
 
+logger = logging.getLogger(__name__)
 
 ACTIVE_RESTRICTED_PATH = dataset_path("active_vehicle_details_restricted.csv")
 ACTIVE_LIVE_PATH = dataset_path("active_vehicle_details.csv")
@@ -161,7 +164,13 @@ def _load_normalized_conditions() -> pd.DataFrame:
         return pd.DataFrame()
     try:
         df = pd.read_csv(NORMALIZED_CONDITIONS_PATH)
-    except Exception:
+    except CSV_READ_ERRORS as exc:
+        logger.warning(
+            "Unreadable normalized conditions %s (%s: %s); condition context will be missing.",
+            NORMALIZED_CONDITIONS_PATH,
+            type(exc).__name__,
+            exc,
+        )
         return pd.DataFrame()
     if "url" not in df.columns or "component_normalized" not in df.columns:
         return pd.DataFrame()

pages/14_CURVE_PIPELINE.py

@@ -15,6 +15,7 @@
     run_autotrader_scrape,
     update_autotrader_queue_status,
 )
+from shared.csv_utils import CSV_READ_ERRORS
 from shared.data_loader import dataset_path
 from shared.styling import display_banner, hero_action_card, inject_global_styles, page_intro, section_heading
 
@@ -46,7 +47,8 @@ def _load_csv(path: Path) -> pd.DataFrame:
         return pd.DataFrame()
     try:
         return pd.read_csv(path, low_memory=False)
-    except Exception:
+    except CSV_READ_ERRORS as exc:
+        st.warning(f"Could not read {path}: {type(exc).__name__}: {exc}")
         return pd.DataFrame()
 
 

pages/18_REPAIR_REVIEW.py

@@ -8,6 +8,7 @@
 import streamlit as st
 from shared.navigation import render_sidebar_navigation
 
+from shared.csv_utils import CSV_READ_ERRORS
 from shared.repair_ai_classifier import AI_SUGGESTIONS_PATH, load_ai_suggestions
 from shared.repair_review import LIVE_QUEUE_PATH
 from shared.styling import clean_html, display_banner, escape_html, inject_global_styles, page_intro
@@ -113,7 +114,8 @@ def load_decisions() -> pd.DataFrame:
         return pd.DataFrame(columns=REVIEW_COLUMNS)
     try:
         df = pd.read_csv(DECISIONS_PATH).fillna("")
-    except Exception:
+    except CSV_READ_ERRORS as exc:
+        st.warning(f"Could not read {DECISIONS_PATH}: {type(exc).__name__}: {exc}")
         return pd.DataFrame(columns=REVIEW_COLUMNS)
     for column in REVIEW_COLUMNS:
         if column not in df.columns:

pages/3_ACTIVE_LISTINGS.py

@@ -128,7 +128,12 @@ async def run_bid_update(links: list[str] | None = None, limit: int | None = Non
         target_links = target_links[:limit_arg]
         limit_arg = None
     with st.spinner("Updating bid and time data..."):
-        df, skipped_urls = await update_bids(input_links=target_links, limit=limit_arg)
+        try:
+            df, skipped_urls = await update_bids(input_links=target_links, limit=limit_arg)
+        except Exception as exc:  # noqa: BLE001 - surface scraper failures in the UI
+            st.error(f"Bid update failed: {type(exc).__name__}: {exc}")
+            st.exception(exc)
+            return
         st.session_state.skipped_urls = skipped_urls
         if not df.empty:
             st.success(f"Updated {len(df)} listings in {CSV_FILE}.")

project_memory/02_state/recent_changes.md

@@ -1,5 +1,7 @@
 # Recent Changes
 
+- 2026-08-09: Made swallowed failures visible across the data-loading, scraping, scheduling, valuation, and Streamlit layers. Broad `except Exception` handlers that returned empty frames, `False`, or partial results were narrowed to the exceptions they actually expect (`CSV_READ_ERRORS`, `OSError`, `ValueError`, `yaml.YAMLError`, `ZoneInfoNotFoundError`) and now log or print the path plus exception before falling back, so an unreadable CSV, curve file, pricing schedule, alert log, or model artifact is reported instead of silently degrading a decision. Two behaviours changed deliberately: `update_bids()` now re-raises after its emergency snapshot instead of returning a partial frame as success (the Active Listings page surfaces the error), and a failed restricted-dataset build now aborts `update_master_database()` instead of printing and continuing. Added `tests/test_error_visibility.py`; full suite, Ruff, readiness, governance, and project-memory checks pass.
+
 - 2026-07-26: Closed the stale Grays sold-tag gap. Restricted-dataset rebuilds now persist refreshed `canonical_tag` / `canonical_reason` assignments back into `sold_cars.csv` without changing its row set or rewriting an unchanged file, and governed `save_curves()` calls immediately rebuild the restricted active/sold datasets so newly supported curves unlock matching historical Grays evidence without waiting for the next daily pipeline.
 - 2026-07-26: Added `scripts/deploy_vps.ps1` for guarded code-only DigitalOcean deployments. It packages the Streamlit/source/config surfaces while explicitly protecting VPS-generated CSV data, curves, artifacts, logs, outputs, and virtual environments; validates Python locally and in a remote staging directory; refuses to deploy over an active scheduled pipeline job unless forced; keeps five rollback snapshots; installs changed requirements; restarts Streamlit; and requires a passing health endpoint before reporting success.
 

scripts/ai_listing_valuation.py

@@ -12,6 +12,7 @@
 
 from scripts.atomic_csv import append_dict_rows_csv_atomic, write_dataframe_csv_atomic
 from shared.auction_model import predict_auction_price
+from shared.csv_utils import CSV_READ_ERRORS
 from shared.data_loader import dataset_path
 from shared.decision_economics import calculate_curve_decision_economics, derive_curve_verdict
 from shared.decision_policy import DecisionPolicyInput, derive_action_label, derive_action_label_from_row
@@ -266,7 +267,7 @@ def load_cached_results() -> pd.DataFrame:
         return pd.DataFrame(columns=REQUIRED_COLUMNS)
     try:
         df = pd.read_csv(AI_RESULTS_PATH)
-    except Exception as exc:
+    except CSV_READ_ERRORS as exc:
         print(f"WARNING: could not read cached AI results {AI_RESULTS_PATH}: {type(exc).__name__}: {exc}")
         return pd.DataFrame(columns=REQUIRED_COLUMNS)
     missing = [column for column in REQUIRED_COLUMNS if column not in df.columns]
@@ -732,7 +733,8 @@ def _dataset_contains_url(path: Path, url: str) -> bool:
         return False
     try:
         df = pd.read_csv(path, usecols=["url"], low_memory=False)
-    except Exception:
+    except CSV_READ_ERRORS as exc:
+        print(f"WARNING: could not read {path} while checking for {url} ({type(exc).__name__}: {exc}).")
         return False
     if df.empty or "url" not in df.columns:
         return False
@@ -1556,7 +1558,7 @@ def apply_platform_risk_adjustments(
 
     try:
         year = int(float(year_raw)) if year_raw not in (None, "") else None
-    except Exception:
+    except (TypeError, ValueError):
         year = None
 
     def add_flag(flag: str) -> None:

scripts/ai_price_analysis.py

@@ -1,3 +1,4 @@
+import logging
 import re
 from dataclasses import dataclass
 from typing import Iterable, List, Optional
@@ -7,8 +8,11 @@
 from difflib import SequenceMatcher
 from pathlib import Path
 
+from shared.csv_utils import CSV_READ_ERRORS
 from shared.data_loader import dataset_path
 
+logger = logging.getLogger(__name__)
+
 
 ACTIVE_PRIMARY_PATH = dataset_path("active_vehicle_details.csv")
 ACTIVE_FALLBACK_PATH = dataset_path("vehicle_static_details.csv")
@@ -184,7 +188,7 @@ def _infer_make_model_from_url(url_value: object) -> tuple[Optional[str], Option
         return None, None
     try:
         parsed = urlparse(str(url_value).strip())
-    except Exception:
+    except ValueError:
         return None, None
     slug = parsed.path.rstrip("/").split("/")[-1]
     if not slug:
@@ -322,9 +326,15 @@ def load_historical_sales(
     for source in extra_sources:
         try:
             normalised = _normalise_sold_dataframe(pd.read_csv(source))
-            dataframes.append(normalised)
-        except Exception:
+        except CSV_READ_ERRORS as exc:
+            logger.warning(
+                "Skipping unreadable sold history source %s (%s: %s); comps may be incomplete.",
+                source,
+                type(exc).__name__,
+                exc,
+            )
             continue
+        dataframes.append(normalised)
 
     if not dataframes:
         return pd.DataFrame()
@@ -872,23 +882,23 @@ def _prepare_match_rows(
     if "url" in df.columns:
         try:
             source_urls = df.loc[subset.index, "url"]
-        except Exception:  # noqa: BLE001
+        except KeyError:
             source_urls = df["url"].head(limit)
 
     def format_price(val):
         if pd.isna(val):
             return "—"
         try:
             return f"${float(val):,.0f}"
-        except Exception:
+        except (TypeError, ValueError):
             return str(val)
 
     def format_odometer(val):
         if pd.isna(val):
             return "—"
         try:
             return f"{int(round(float(val))):,} km"
-        except Exception:
+        except (TypeError, ValueError):
             text = str(val).strip()
             if not text:
                 return "—"
@@ -900,15 +910,15 @@ def format_year_with_delta(year_val, delta_val):
         try:
             base_year = int(round(float(year_val)))
             base_text = str(base_year)
-        except Exception:
+        except (TypeError, ValueError):
             base_text = str(year_val).strip() or "—"
         if not include_year_delta:
             return base_text
         if pd.isna(delta_val) or delta_val in (None, 0):
             return f"{base_text} (+/-1y fallback)"
         try:
             delta_int = int(round(float(delta_val)))
-        except Exception:
+        except (TypeError, ValueError):
             return f"{base_text} (+/-1y fallback)"
         if delta_int == 0:
             return f"{base_text} (+/-1y fallback)"
@@ -937,7 +947,7 @@ def format_reauction_count(value):
             return ""
         try:
             return int(round(float(value)))
-        except Exception:
+        except (TypeError, ValueError):
             return str(value)
 
     if "reauction_group_size" in subset.columns:

scripts/extract_vehicle_details.py

@@ -18,6 +18,7 @@
 
     sys.path.append(str(Path(__file__).resolve().parent.parent))
     from scripts.atomic_csv import append_dataframe_csv_atomic, write_dataframe_csv_atomic
+    from shared.csv_utils import CSV_READ_ERRORS
     from shared.data_loader import dataset_path
     from shared.schema import ACTIVE_DETAIL_SCHEMA, SOLD_RAW_SCRAPE_COLUMNS, STATIC_VEHICLE_SCHEMA
     from shared.sold_cleaning import (
@@ -256,7 +257,11 @@ def load_make_whitelist(existing_df: pd.DataFrame) -> set[str]:
                 for make in sold_df["make"].dropna().unique().tolist()
                 if str(make).strip()
             }
-        except Exception:
+        except CSV_READ_ERRORS as exc:
+            print(
+                f"WARNING: could not build make whitelist from {sold_path} "
+                f"({type(exc).__name__}: {exc}); falling back to existing listings."
+            )
             whitelist = set()
     if not whitelist and "make" in existing_df.columns:
         whitelist = {
@@ -723,10 +728,10 @@ def _parse_literal(raw: str) -> Any:
     normalized = _normalize_js_literals(raw)
     try:
         return ast.literal_eval(normalized)
-    except Exception:
+    except (ValueError, SyntaxError, MemoryError, RecursionError):
         try:
             return json.loads(raw)
-        except Exception:
+        except ValueError:
             return None
 
 

scripts/process_curve_candidates.py

@@ -22,12 +22,14 @@
     from scripts.atomic_csv import write_dataframe_csv_atomic
     from scripts.curve_validator import build_curve_warnings
     from shared.canonical_tagging import tag_dataframe
+    from shared.csv_utils import CSV_READ_ERRORS
     from shared.curves import CURVE_COLUMNS, resolve_curve_canonical_tag
     from shared.data_loader import dataset_path
 else:  # pragma: no cover
     from scripts.atomic_csv import write_dataframe_csv_atomic
     from scripts.curve_validator import build_curve_warnings
     from shared.canonical_tagging import tag_dataframe
+    from shared.csv_utils import CSV_READ_ERRORS
     from shared.curves import CURVE_COLUMNS, resolve_curve_canonical_tag
     from shared.data_loader import dataset_path
 
@@ -222,7 +224,8 @@ def load_carsales_apify_market(path: Path | None = None) -> pd.DataFrame:
         return pd.DataFrame()
     try:
         df = pd.read_csv(csv_path, low_memory=False)
-    except Exception:
+    except CSV_READ_ERRORS as exc:
+        print(f"WARNING: unreadable Autotrader market file {csv_path} ({type(exc).__name__}: {exc}).")
         return pd.DataFrame()
     if df.empty:
         return pd.DataFrame()

scripts/run_grays_pipeline_loop.py

@@ -13,8 +13,10 @@
 
 if __package__ in (None, ""):
     sys.path.append(str(Path(__file__).resolve().parent.parent))
+    from shared.csv_utils import CSV_READ_ERRORS
     from shared.data_loader import dataset_path
 else:  # pragma: no cover
+    from shared.csv_utils import CSV_READ_ERRORS
     from shared.data_loader import dataset_path
 
 
@@ -53,7 +55,8 @@ def _read_row_count(path: Path) -> int:
         return 0
     try:
         df = pd.read_csv(path, low_memory=False)
-    except Exception:
+    except CSV_READ_ERRORS as exc:
+        print(f"WARNING: could not count rows in {path} ({type(exc).__name__}: {exc}); treating as 0.")
         return 0
     return len(df)