diff --git a/.github/workflows/ocr-regression-degraded.yml b/.github/workflows/ocr-regression-degraded.yml new file mode 100644 index 00000000..fa65c101 --- /dev/null +++ b/.github/workflows/ocr-regression-degraded.yml @@ -0,0 +1,82 @@ +name: OCR Regression Test (Degraded) + +on: + push: + paths: + - 'app/ai-service/services/ocr.py' + - 'app/ai-service/services/preprocessing.py' + - 'app/ai-service/regression_harness/dataset/degraded/**' + - 'app/ai-service/regression_harness/**' + branches: [ main, develop ] + pull_request: + paths: + - 'app/ai-service/services/ocr.py' + - 'app/ai-service/services/preprocessing.py' + - 'app/ai-service/regression_harness/dataset/degraded/**' + - 'app/ai-service/regression_harness/**' + branches: [ main ] + workflow_dispatch: + +jobs: + regression-degraded: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt-get install -y tesseract-ocr libtesseract-dev + + - name: Install Python Dependencies + working-directory: ./app/ai-service + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install Pillow pytesseract + + - name: Run OCR Regression Harness (degraded) + working-directory: ./app/ai-service + run: | + set -euo pipefail + export PYTHONPATH=$PYTHONPATH:. + # We require: + # - >= 90% of samples pass the exact-field matching gate + # - >= 60% reported accuracy + python regression_harness/cli.py \ + --dataset regression_harness/dataset/degraded/ground_truth.json \ + --output ocr_degraded_report.json \ + --threshold 0.8 + + python - <<'PY' + import json + with open('ocr_degraded_report.json','r') as f: + report = json.load(f) + summary = report.get('summary', {}) + total = summary.get('total', 0) + passed = summary.get('passed', 0) + accuracy = float(summary.get('accuracy', 0.0)) + pass_ratio = (passed/total) if total else 0.0 + print('Degraded regression summary:', {'total': total, 'passed': passed, 'pass_ratio': pass_ratio, 'accuracy': accuracy}) + if pass_ratio < 0.9: + raise SystemExit(f'FAILED: pass_ratio {pass_ratio:.3f} < 0.9') + if accuracy < 0.6: + raise SystemExit(f'FAILED: accuracy {accuracy:.3f}% < 0.6%') + PY + + - name: Upload Regression Report + if: always() + uses: actions/upload-artifact@v4 + with: + name: ocr-regression-degraded-report + path: app/ai-service/ocr_degraded_report.json + retention-days: 14 + diff --git a/app/ai-service/metrics.py b/app/ai-service/metrics.py index a94e5342..a62ada79 100644 --- a/app/ai-service/metrics.py +++ b/app/ai-service/metrics.py @@ -42,6 +42,14 @@ CIRCUIT_STATE_HALF_OPEN = 1 CIRCUIT_STATE_OPEN = 2 +# Human-readable labels for the encoded gauge values. This keeps the metric +# values and the Grafana/operational mapping aligned with the same constants. +CIRCUIT_STATE_LABELS = { + CIRCUIT_STATE_CLOSED: 'CLOSED', + CIRCUIT_STATE_HALF_OPEN: 'HALF_OPEN', + CIRCUIT_STATE_OPEN: 'OPEN', +} + def set_circuit_state(breaker_name: str, state_value: int) -> None: """Helper to update the circuit-state gauge from anywhere.""" diff --git a/app/ai-service/regression_harness/cli.py b/app/ai-service/regression_harness/cli.py index 58837154..94a86f8e 100644 --- a/app/ai-service/regression_harness/cli.py +++ b/app/ai-service/regression_harness/cli.py @@ -1,7 +1,15 @@ import os +import sys import json import argparse from typing import List +# Ensure this script can be executed directly regardless of CWD / PYTHONPATH. +# When running as: python app/ai-service/regression_harness/cli.py +# we want to treat `app/ai-service` as the import root. +import_path_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +if import_path_root not in sys.path: + sys.path.insert(0, import_path_root) + from regression_harness.models import EvaluationSample, BoundingBox from regression_harness.evaluator import OCREvaluator @@ -54,11 +62,13 @@ def main(): parser.add_argument("--dataset", default="regression_harness/dataset/ground_truth.json", help="Path to ground truth JSON") parser.add_argument("--output", help="Path to save JSON report") parser.add_argument("--threshold", type=float, default=0.8, help="Confidence threshold") - + parser.add_argument("--min_pass_ratio", type=float, default=None, help="If set, CI can enforce minimum pass ratio (0-1).") + args = parser.parse_args() base_dir = os.path.dirname(os.path.abspath(__file__)) - # Adjust base_dir if it's currently inside regression_harness + # Ensure args.dataset paths work regardless of where this script is run from. + # Default expects to be relative to app/ai-service. if base_dir.endswith("regression_harness"): base_dir = os.path.dirname(base_dir) # We want base_dir to be app/ai-service @@ -81,7 +91,12 @@ def main(): json.dump(report.to_dict(), f, indent=2) print(f"Report saved to {args.output}") - if report.failed_samples > 0: + if args.min_pass_ratio is not None: + pass_ratio = (report.passed_samples / report.total_samples) if report.total_samples > 0 else 0 + print(f"Min pass ratio requirement: {args.min_pass_ratio:.2f}, actual: {pass_ratio:.2f}") + if pass_ratio < args.min_pass_ratio: + exit(1) + elif report.failed_samples > 0: exit(1) if __name__ == "__main__": diff --git a/app/ai-service/regression_harness/dataset/degraded/README.md b/app/ai-service/regression_harness/dataset/degraded/README.md new file mode 100644 index 00000000..745423e4 --- /dev/null +++ b/app/ai-service/regression_harness/dataset/degraded/README.md @@ -0,0 +1,12 @@ +# Degraded regression dataset + +This dataset contains intentionally degraded versions of the golden `sample_001.png` fixture to validate OCR robustness against: +- 90° rotations +- low contrast +- blur +- low resolution +- a faint watermark overlay + +Images live in `documents/`. +Ground truth lives in `ground_truth.json`. + diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_blur2_lowc.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_blur2_lowc.png new file mode 100644 index 00000000..ab1d3cba Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_blur2_lowc.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_blur4_lowc.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_blur4_lowc.png new file mode 100644 index 00000000..19c25972 Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_blur4_lowc.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowc1.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowc1.png new file mode 100644 index 00000000..59a9972b Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowc1.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowc2.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowc2.png new file mode 100644 index 00000000..de7d31d0 Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowc2.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowres.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowres.png new file mode 100644 index 00000000..c5a55421 Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowres.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowres2.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowres2.png new file mode 100644 index 00000000..83e095fa Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_lowres2.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_orig.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_orig.png new file mode 100644 index 00000000..142efe26 Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_orig.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_rot180.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_rot180.png new file mode 100644 index 00000000..4a176d8e Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_rot180.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_rot270.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_rot270.png new file mode 100644 index 00000000..4f875c91 Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_rot270.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_rot90.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_rot90.png new file mode 100644 index 00000000..5e70ba36 Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_rot90.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_watermark30.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_watermark30.png new file mode 100644 index 00000000..ab5daa99 Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_watermark30.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_watermark60.png b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_watermark60.png new file mode 100644 index 00000000..ab5daa99 Binary files /dev/null and b/app/ai-service/regression_harness/dataset/degraded/documents/sample_001_watermark60.png differ diff --git a/app/ai-service/regression_harness/dataset/degraded/ground_truth.json b/app/ai-service/regression_harness/dataset/degraded/ground_truth.json new file mode 100644 index 00000000..876d524e --- /dev/null +++ b/app/ai-service/regression_harness/dataset/degraded/ground_truth.json @@ -0,0 +1,268 @@ +{ + "samples": [ + { + "id": "sample_001_orig", + "image_path": "documents/sample_001_orig.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "none" + } + }, + { + "id": "sample_001_rot90", + "image_path": "documents/sample_001_rot90.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "rot90" + } + }, + { + "id": "sample_001_rot180", + "image_path": "documents/sample_001_rot180.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "rot180" + } + }, + { + "id": "sample_001_rot270", + "image_path": "documents/sample_001_rot270.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "rot270" + } + }, + { + "id": "sample_001_lowc1", + "image_path": "documents/sample_001_lowc1.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "low_contrast_0.5" + } + }, + { + "id": "sample_001_lowc2", + "image_path": "documents/sample_001_lowc2.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "low_contrast_0.3" + } + }, + { + "id": "sample_001_blur2_lowc", + "image_path": "documents/sample_001_blur2_lowc.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "blur2" + } + }, + { + "id": "sample_001_blur4_lowc", + "image_path": "documents/sample_001_blur4_lowc.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "blur4" + } + }, + { + "id": "sample_001_lowres_07", + "image_path": "documents/sample_001_lowres.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "lowres_0.7" + } + }, + { + "id": "sample_001_lowres_05", + "image_path": "documents/sample_001_lowres2.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "lowres_0.5" + } + }, + { + "id": "sample_001_watermark30", + "image_path": "documents/sample_001_watermark30.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "watermark_30" + } + }, + { + "id": "sample_001_watermark60", + "image_path": "documents/sample_001_watermark60.png", + "expected_fields": { + "name": "John Doe", + "date_of_birth": "15 Jan 1990", + "id_number": "AB123456" + }, + "expected_bboxes": { + "name": { + "x": 100, + "y": 200, + "width": 300, + "height": 50 + } + }, + "metadata": { + "document_type": "id_card", + "language": "en", + "degradation": "watermark_60" + } + } + ] +} \ No newline at end of file diff --git a/app/ai-service/regression_harness/dataset/degraded/placeholder.txt b/app/ai-service/regression_harness/dataset/degraded/placeholder.txt new file mode 100644 index 00000000..5493b7c9 --- /dev/null +++ b/app/ai-service/regression_harness/dataset/degraded/placeholder.txt @@ -0,0 +1,2 @@ +degraded dataset will be added here. + diff --git a/app/ai-service/services/ocr.py b/app/ai-service/services/ocr.py index 494f83fd..a48b08bc 100644 --- a/app/ai-service/services/ocr.py +++ b/app/ai-service/services/ocr.py @@ -97,38 +97,73 @@ def process_image(self, image: Image.Image) -> OCRResult: start_time = time.time() - preprocessed = self.preprocessor.preprocess( - image, threshold_method="otsu", denoise=True - ) - - if preprocessed.size[0] == 0 or preprocessed.size[1] == 0: - return OCRResult( - fields={}, - raw_text="", - processing_time_ms=int((time.time() - start_time) * 1000), + # Robustness for rotated / degraded images: + # Try multiple orientations and pick the candidate with the highest + # number of detected fields (then confidence). + candidates = [0, 90, 180, 270] + best = None # (score_fields, score_conf, OCRResult) + + for ang in candidates: + rotated = image.rotate(ang, expand=True) if ang else image + preprocessed = self.preprocessor.preprocess( + rotated, threshold_method="otsu", denoise=True ) - tesseract_data = self._run_tesseract(preprocessed) + if preprocessed.size[0] == 0 or preprocessed.size[1] == 0: + continue + + tesseract_data = self._run_tesseract(preprocessed) + + raw_text = tesseract_data.get("text", "") + if isinstance(raw_text, list): + raw_text = " ".join(str(t) for t in raw_text if t) + raw_text = str(raw_text) if raw_text else "" + + fields = self.field_detector.detect_fields(raw_text) - raw_text = tesseract_data.get("text", "") - if isinstance(raw_text, list): - raw_text = " ".join(str(t) for t in raw_text if t) - raw_text = str(raw_text) if raw_text else "" + # Update per-field confidence (character-level aggregation) + total_conf = 0.0 + for field_name, field_match in fields.items(): + field_chars = self._extract_field_chars( + tesseract_data, field_match.value + ) + field_match.confidence = self.field_detector.aggregate_confidence( + field_chars + ) + total_conf += field_match.confidence - fields = self.field_detector.detect_fields(raw_text) + score_fields = len(fields) + score_conf = total_conf - for field_name, field_match in fields.items(): - field_chars = self._extract_field_chars(tesseract_data, field_match.value) - field_match.confidence = self.field_detector.aggregate_confidence( - field_chars + ocr_result = OCRResult( + fields=fields, + raw_text=raw_text, + processing_time_ms=0, + ) + + if best is None: + best = (score_fields, score_conf, ocr_result) + else: + # Prefer more fields; tie-break by confidence + if (score_fields, score_conf) > (best[0], best[1]): + best = (score_fields, score_conf, ocr_result) + + if best is None: + return OCRResult( + fields={}, + raw_text="", + processing_time_ms=int((time.time() - start_time) * 1000), ) latency = time.time() - start_time metrics.PIPELINE_STEP_LATENCY.labels(step_name='ocr').observe(latency) + best_fields = best[2].fields + best_raw_text = best[2].raw_text + return OCRResult( - fields=fields, - raw_text=raw_text, + fields=best_fields, + raw_text=best_raw_text, processing_time_ms=int(latency * 1000), ) diff --git a/app/ai-service/services/preprocessing.py b/app/ai-service/services/preprocessing.py index c6681f3b..091be26e 100644 --- a/app/ai-service/services/preprocessing.py +++ b/app/ai-service/services/preprocessing.py @@ -56,8 +56,15 @@ def preprocess( threshold_method: str = "otsu", denoise: bool = True, ) -> Image.Image: + """Preprocess an image for OCR. + + Robustness goals: + - low-contrast documents (contrast normalization + CLAHE) + - blurred images (light denoise) + - rotated images (handled by OCRService via rotated candidates; preprocess stays deterministic) + """ start_time = time.time() - + try: if image.size[0] == 0 or image.size[1] == 0: return image.convert("L") @@ -68,8 +75,19 @@ def preprocess( if denoise: gray = self.denoise(gray) + # Contrast normalization for low-contrast images + gray_np = self.image_to_numpy(gray) + clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) + gray_np = clahe.apply(gray_np) + + gray = self.numpy_to_image(gray_np) + thresholded = self.apply_threshold(gray, method=threshold_method) + # Final cleanup: small morphological closing to improve low-contrast/blur robustness + kernel = np.ones((3, 3), np.uint8) + thresholded = cv2.morphologyEx(thresholded, cv2.MORPH_CLOSE, kernel, iterations=1) + return thresholded finally: latency = time.time() - start_time diff --git a/app/ai-service/tests/test_circuit_breaker.py b/app/ai-service/tests/test_circuit_breaker.py index 0de524ba..8fb2f978 100644 --- a/app/ai-service/tests/test_circuit_breaker.py +++ b/app/ai-service/tests/test_circuit_breaker.py @@ -140,6 +140,11 @@ def _sample(name: str, labels: dict) -> float: class TestCircuitBreakerMetrics: """Verify that CircuitBreaker publishes the metrics defined in metrics.py.""" + def test_circuit_state_labels_use_named_constants(self): + assert metrics.CIRCUIT_STATE_LABELS[metrics.CIRCUIT_STATE_CLOSED] == "CLOSED" + assert metrics.CIRCUIT_STATE_LABELS[metrics.CIRCUIT_STATE_HALF_OPEN] == "HALF_OPEN" + assert metrics.CIRCUIT_STATE_LABELS[metrics.CIRCUIT_STATE_OPEN] == "OPEN" + def test_initial_state_is_published(self): # Using a fresh breaker name ensures labels don't collide with other tests. CircuitBreaker("metrics-initial", failure_threshold=1, recovery_timeout=0.1) diff --git a/docs/observability/ai-service-grafana.json b/docs/observability/ai-service-grafana.json new file mode 100644 index 00000000..eec681d3 --- /dev/null +++ b/docs/observability/ai-service-grafana.json @@ -0,0 +1,85 @@ +{ + "title": "AI Service Circuit Breaker States", + "description": "Grafana panel showing provider circuit breaker state with inline mapping: 0=CLOSED, 1=HALF_OPEN, 2=OPEN.", + "panels": [ + { + "type": "stat", + "title": "Circuit breaker state by provider", + "targets": [ + { + "expr": "circuit_breaker_state{breaker_name=~\"openai|groq|test\"}", + "legendFormat": "{{breaker_name}}" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "unknown" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "color-background" + } + ] + } + ] + }, + "transformations": [ + { + "id": "labelsToFields", + "options": {} + } + ], + "datasource": "Prometheus", + "options": { + "graphMode": "area", + "colorMode": "background", + "justifyMode": "center" + }, + "description": "State mapping documented inline: 0=CLOSED, 1=HALF_OPEN, 2=OPEN. Use the same constants as metrics.py: CIRCUIT_STATE_CLOSED, CIRCUIT_STATE_HALF_OPEN, CIRCUIT_STATE_OPEN." + } + ] +} \ No newline at end of file