Harden labeled evaluation and tracking edge cases

beltmap/advanced_quality.py

@@ -25,6 +25,7 @@
 from numpy.typing import ArrayLike, NDArray
 
 FloatArray = NDArray[np.floating]
+REVIEWED_GROUND_TRUTH_STATUS = "reviewed_ground_truth"
 
 
 @dataclass(frozen=True)
@@ -434,12 +435,9 @@ def detection_boxes_by_frame(output_dir: Path) -> dict[int, list[dict[str, float
             frame = finite_int(row["frame_index"])
             if frame is None:
                 continue
-            box = {
-                "top": float(row["bbox_top"]),
-                "left": float(row["bbox_left"]),
-                "bottom": float(row["bbox_bottom"]),
-                "right": float(row["bbox_right"]),
-            }
+            box = _parse_detection_box(row)
+            if box is None:
+                continue
         except (KeyError, TypeError, ValueError):
             continue
         grouped.setdefault(frame, []).append(box)
@@ -456,6 +454,8 @@ def load_real_label_boxes(path: Path) -> dict[int, list[dict[str, float]]]:
     """
 
     data = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(data, dict):
+        _validate_reviewed_truth_status(data)
     frames = data.get("frames") if isinstance(data, dict) else None
     if not isinstance(frames, list):
         raise ValueError("label JSON must contain a 'frames' list")
@@ -464,13 +464,68 @@ def load_real_label_boxes(path: Path) -> dict[int, list[dict[str, float]]]:
         frame_index = finite_int(frame.get("frame_index"))
         if frame_index is None:
             continue
-        boxes = []
+        boxes: list[dict[str, float]] = []
         for box in frame.get("boxes", []):
-            boxes.append({key: float(box[key]) for key in ("top", "left", "bottom", "right")})
+            boxes.append(_parse_real_label_box(box))
         result[frame_index] = boxes
     return result
 
 
+def _review_flag_is_true(value: Any) -> bool:
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        if normalized in {"", "0", "false", "no", "n"}:
+            return False
+        if normalized in {"1", "true", "yes", "y"}:
+            return True
+    return bool(value)
+
+
+def _validate_reviewed_truth_status(data: dict[str, Any]) -> None:
+    status = data.get("status")
+    requires_manual_review = data.get("requires_manual_review")
+    if status is None and requires_manual_review is None:
+        return
+    if status != REVIEWED_GROUND_TRUTH_STATUS or _review_flag_is_true(
+        requires_manual_review,
+    ):
+        raise ValueError(
+            "label JSON is not reviewed ground truth; set status="
+            f"{REVIEWED_GROUND_TRUTH_STATUS!r} and requires_manual_review=false "
+            "only after all scored frames have been reviewed"
+        )
+
+
+def _parse_real_label_box(box: Mapping[str, Any]) -> dict[str, float]:
+    try:
+        top = float(box["top"])
+        left = float(box["left"])
+        bottom = float(box["bottom"])
+        right = float(box["right"])
+    except (KeyError, TypeError, ValueError) as exc:
+        raise ValueError("label boxes must contain numeric top/left/bottom/right") from exc
+    if not all(math.isfinite(value) for value in (top, left, bottom, right)):
+        raise ValueError("label box coordinates must be finite")
+    if bottom <= top or right <= left:
+        raise ValueError("label boxes must have positive half-open area")
+    return {"top": top, "left": left, "bottom": bottom, "right": right}
+
+
+def _parse_detection_box(row: Mapping[str, Any]) -> dict[str, float] | None:
+    values = [
+        finite_float(row.get(key))
+        for key in ("bbox_top", "bbox_left", "bbox_bottom", "bbox_right")
+    ]
+    if any(value is None for value in values):
+        return None
+    top, left, bottom, right = values
+    assert top is not None and left is not None
+    assert bottom is not None and right is not None
+    if bottom <= top or right <= left:
+        return None
+    return {"top": top, "left": left, "bottom": bottom, "right": right}
+
+
 def evaluate_real_detections(output_dir: Path, labels_path: Path, *, iou_threshold: float = 0.5) -> RealLabelMetrics:
     """Evaluate BeltMap detections against sparse real-data annotations."""
 

beltmap/benchmark.py

@@ -257,6 +257,8 @@ def bbox_from_detection(row: dict[str, Any]) -> dict[str, float] | None:
     top, left, bottom, right = values
     assert top is not None and left is not None
     assert bottom is not None and right is not None
+    if bottom <= top or right <= left:
+        return None
     box = {"top": top, "left": left, "bottom": bottom, "right": right}
     y = finite_float(row.get("y"))
     x = finite_float(row.get("x"))
@@ -813,7 +815,11 @@ def detection_metrics(
 
     truth_by_frame = group_truth_boxes(truth)
     pred_by_frame = group_detection_boxes(detection_rows)
-    frame_indices = sorted((scored_frames or set()) | set(truth_by_frame) | set(pred_by_frame))
+    frame_indices = (
+        sorted(set(truth_by_frame) | set(pred_by_frame))
+        if scored_frames is None
+        else sorted(scored_frames)
+    )
 
     true_positives = 0
     false_positives = 0
@@ -856,8 +862,8 @@ def detection_metrics(
     return {
         "available": bool(frame_indices),
         "iou_threshold": iou_threshold,
-        "truth_boxes": sum(len(items) for items in truth_by_frame.values()),
-        "predicted_boxes": sum(len(items) for items in pred_by_frame.values()),
+        "truth_boxes": sum(len(truth_by_frame.get(frame, [])) for frame in frame_indices),
+        "predicted_boxes": sum(len(pred_by_frame.get(frame, [])) for frame in frame_indices),
         "true_positives": true_positives,
         "false_positives": false_positives,
         "false_negatives": false_negatives,

beltmap/bootstrap_ci.py

@@ -212,7 +212,7 @@ def labeled_frame_outcomes(
     pred_by_frame = group_detection_boxes(
         [dict(row) for row in detection_rows if source_frame_index(dict(row)) in scored_frames]
     )
-    frame_indices = sorted(scored_frames | set(truth_by_frame) | set(pred_by_frame))
+    frame_indices = sorted(scored_frames)
     outcomes: list[LabeledFrameOutcome] = []
 
     for frame_index in frame_indices:

beltmap/compare_runs.py

@@ -811,7 +811,12 @@ def detection_froc_curve(
         if score_field is not None or not detection_rows
         else incomplete_score_row_count(detection_rows, score_fields=score_fields)
     )
-    empty_metrics = detection_metrics([], truth, iou_threshold=iou_threshold)
+    empty_metrics = detection_metrics(
+        [],
+        truth,
+        iou_threshold=iou_threshold,
+        scored_frames=scored_frames,
+    )
     truth_boxes = int(empty_metrics.get("truth_boxes") or 0)
     points = [
         froc_point_from_metrics(
@@ -834,7 +839,12 @@ def detection_froc_curve(
                 if (score := finite_float(row.get(score_field))) is not None
                 and score >= threshold
             ]
-            metrics = detection_metrics(kept_rows, truth, iou_threshold=iou_threshold)
+            metrics = detection_metrics(
+                kept_rows,
+                truth,
+                iou_threshold=iou_threshold,
+                scored_frames=scored_frames,
+            )
             points.append(
                 froc_point_from_metrics(
                     metrics,
@@ -843,7 +853,12 @@ def detection_froc_curve(
                 )
             )
     elif detection_rows:
-        metrics = detection_metrics(detection_rows, truth, iou_threshold=iou_threshold)
+        metrics = detection_metrics(
+            detection_rows,
+            truth,
+            iou_threshold=iou_threshold,
+            scored_frames=scored_frames,
+        )
         points.append(
             froc_point_from_metrics(
                 metrics,
@@ -947,6 +962,17 @@ def metadata_or_count(data: RunData, key: str, rows: list[Any]) -> int | None:
     return int(value)
 
 
+def metadata_count_or_none(data: RunData, key: str) -> int | None:
+    """Return a validated metadata count, or ``None`` when it is absent."""
+
+    if key not in data.metadata:
+        return None
+    value = finite_float(data.metadata.get(key))
+    if value is None or value < 0 or not value.is_integer():
+        raise ValueError(f"metadata {key!r} must be a non-negative integer-like value")
+    return int(value)
+
+
 def empty_labeled_metrics() -> dict[str, Any]:
     """Return blank labeled-target metrics for proxy-only comparisons."""
 
@@ -1032,7 +1058,7 @@ def summarize_run(
         "n_images": metadata_or_count(data, "n_images", data.detections_per_frame),
         "detection_threshold": detection_threshold,
         "n_detections": metadata_or_count(data, "n_detections", data.detections),
-        "n_tracks": data.metadata.get("n_tracks"),
+        "n_tracks": metadata_count_or_none(data, "n_tracks"),
         "n_velocity_estimates": metadata_or_count(data, "n_velocity_estimates", data.velocities),
         "n_filtered_velocity_estimates": metadata_or_count(
             data,

beltmap/registration_quality.py

@@ -55,16 +55,26 @@ def validate(self) -> None:
             raise ValueError(
                 "RegistrationQualityGateConfig.action must already be normalized"
             )
-        if self.min_score is not None and self.min_score < 0:
-            raise ValueError("min_score must be non-negative when set")
-        if self.min_loss_gap_ratio is not None and self.min_loss_gap_ratio < 0:
-            raise ValueError("min_loss_gap_ratio must be non-negative when set")
-        if self.max_uncertainty_px is not None and self.max_uncertainty_px < 0:
-            raise ValueError("max_uncertainty_px must be non-negative when set")
-        if self.max_abs_correction_px is not None and self.max_abs_correction_px < 0:
-            raise ValueError("max_abs_correction_px must be non-negative when set")
+        _validate_optional_non_negative(self.min_score, "min_score")
+        _validate_optional_non_negative(
+            self.min_loss_gap_ratio,
+            "min_loss_gap_ratio",
+        )
+        _validate_optional_non_negative(
+            self.max_uncertainty_px,
+            "max_uncertainty_px",
+        )
+        _validate_optional_non_negative(
+            self.max_abs_correction_px,
+            "max_abs_correction_px",
+        )
+        _require_finite(self.noise_inflation_factor, "noise_inflation_factor")
         if self.noise_inflation_factor < 1.0:
             raise ValueError("noise_inflation_factor must be at least 1")
+        _require_finite(
+            self.uncertainty_inflation_scale,
+            "uncertainty_inflation_scale",
+        )
         if self.uncertainty_inflation_scale < 0.0:
             raise ValueError("uncertainty_inflation_scale must be non-negative")
 
@@ -261,3 +271,16 @@ def _at_most(value: float | None, threshold: float) -> bool:
 
 def _csv_float(value: float | None) -> float | str:
     return "" if value is None else float(value)
+
+
+def _validate_optional_non_negative(value: float | None, name: str) -> None:
+    if value is None:
+        return
+    _require_finite(value, name)
+    if value < 0:
+        raise ValueError(f"{name} must be non-negative when set")
+
+
+def _require_finite(value: float, name: str) -> None:
+    if not np.isfinite(float(value)):
+        raise ValueError(f"{name} must be finite")

beltmap/tracking.py

@@ -655,8 +655,13 @@ def estimate_particle_velocities_vs_belt(
             continue
         ys = np.asarray([d.y for d in track.detections], dtype=np.float64)
         xs = np.asarray([d.x for d in track.detections], dtype=np.float64)
-        vy = _fit_slope(frames, ys, method=fit_method)
-        vx = _fit_slope(frames, xs, method=fit_method)
+        try:
+            vy = _fit_slope(frames, ys, method=fit_method)
+            vx = _fit_slope(frames, xs, method=fit_method)
+        except ValueError:
+            continue
+        if not np.isfinite(vy) or not np.isfinite(vx):
+            continue
         velocities.append(
             ParticleVelocity(
                 track_id=track.track_id,
@@ -1385,6 +1390,11 @@ def _fit_slope(times: FloatArray, values: FloatArray, *, method: str) -> float:
 
 
 def _linear_slope(times: FloatArray, values: FloatArray) -> float:
+    finite = np.isfinite(times) & np.isfinite(values)
+    times = np.asarray(times[finite], dtype=np.float64)
+    values = np.asarray(values[finite], dtype=np.float64)
+    if np.unique(times).size < 2:
+        raise ValueError("at least two distinct frame indices are required")
     centered_times = times - float(np.mean(times))
     denominator = float(np.sum(np.square(centered_times)))
     if denominator <= 0:

beltmap/visual_qc.py

@@ -268,6 +268,8 @@ def parse_detection_records(rows: Iterable[dict[str, Any]]) -> list[DetectionRec
         assert x is not None and y is not None
         assert top is not None and left is not None
         assert bottom is not None and right is not None
+        if bottom <= top or right <= left:
+            continue
         records.append(
             DetectionRecord(
                 frame_index=frame_index,

scripts/apply_beltmap_to_images.py

@@ -73,6 +73,8 @@ def env_int(name: str, default: int, minimum: int | None = None) -> int:
 def env_float(name: str, default: float, minimum: float | None = None) -> float:
     value = os.getenv(name, "").strip()
     parsed = default if value == "" else float(value)
+    if not math.isfinite(parsed):
+        raise ValueError(f"{name} must be finite")
     if minimum is not None and parsed < minimum:
         raise ValueError(f"{name}={parsed} is below minimum {minimum}")
     return parsed
@@ -87,6 +89,8 @@ def env_optional_float(
     if value == "":
         return default
     parsed = float(value)
+    if not math.isfinite(parsed):
+        raise ValueError(f"{name} must be finite")
     if minimum is not None and parsed < minimum:
         raise ValueError(f"{name}={parsed} is below minimum {minimum}")
     return parsed
@@ -340,10 +344,20 @@ def resolve_velocity_frame_unit(frame_stride: int) -> str:
 
 def resolve_supplied_velocity(velocity_spec: str, frame_stride: int) -> tuple[float, str, float]:
     raw_velocity = float(velocity_spec)
+    if not math.isfinite(raw_velocity):
+        raise ValueError(
+            "BELT_VELOCITY_PX_PER_FRAME must be finite or the literal value 'auto'"
+        )
     frame_unit = resolve_velocity_frame_unit(frame_stride)
     if frame_unit == SOURCE_FRAME_VELOCITY_UNIT:
-        return raw_velocity * frame_stride, frame_unit, raw_velocity
-    return raw_velocity, frame_unit, raw_velocity
+        effective_velocity = raw_velocity * frame_stride
+    else:
+        effective_velocity = raw_velocity
+    if not math.isfinite(effective_velocity):
+        raise ValueError(
+            "BELT_VELOCITY_PX_PER_FRAME must produce a finite selected-frame velocity"
+        )
+    return effective_velocity, frame_unit, raw_velocity
 
 
 def optional_positive_int(name: str) -> int | None: