diff --git a/README.md b/README.md index 0dccde7..7920e7e 100644 --- a/README.md +++ b/README.md @@ -1,934 +1,212 @@ -# SENTIGON +
-**Agentic Physical Security Intelligence Platform** +# πŸ›‘οΈ SENTIGON -Sentinel AI is a production-grade, full-stack security operations platform that unifies real-time video analytics, multi-modal threat detection, autonomous AI agents, and forensic investigation into a single Security Operations Center (SOC). The system processes live camera feeds through a layered perception-reasoning-action agent architecture, correlates events across cameras, sensors, and access control systems, and provides operators with an intelligent command interface for physical security management. +### The Agentic Security Operations Center that runs **100% on your own hardware** ---- - -## Table of Contents - -1. [System Architecture](#system-architecture) -2. [Architecture Diagram](#architecture-diagram) -3. [Technology Stack](#technology-stack) -4. [Project Structure](#project-structure) -5. [Agent System](#agent-system) -6. [Backend Services](#backend-services) -7. [API Surface](#api-surface) -8. [Frontend Application](#frontend-application) -9. [Database Schema](#database-schema) -10. [Threat Detection Engine](#threat-detection-engine) -11. [Integration Layer](#integration-layer) -12. [Quick Start](#quick-start) -13. [Configuration](#configuration) -14. [RBAC and Authentication](#rbac-and-authentication) -15. [Deployment](#deployment) - ---- - -## System Architecture - - -sentigon-architecture - - -Sentinel AI follows a layered architecture with four principal tiers: - -**Data Ingestion Layer**: Camera feeds (RTSP, USB, ONVIF), IoT sensors (MQTT, Modbus), PACS card readers, alarm panels, and environmental monitors continuously stream data into the platform. - -**Perception Layer**: YOLOv8 performs real-time object detection with ByteTrack multi-object tracking. CLIP generates visual embeddings for semantic search. Audio classifiers detect gunshots, glass breaking, and screams. Each feed is processed at up to 15 FPS with 30-frame ring buffers per camera. - -**Reasoning Layer**: Google Gemini provides scene understanding, forensic analysis, and natural language threat assessment. A threat engine with 165+ hybrid signatures (YOLO + Gemini keyword matching) classifies detections. Baseline learning services establish per-zone, per-time-slot normality profiles. Causal reasoning and counterfactual engines analyze incident chains. - -**Action Layer**: Autonomous response services trigger alerts, lockdowns, dispatch assignments, and multi-channel notifications. A human-in-the-loop (HITL) approval system gates high-impact actions. All actions are audited with full chain-of-custody logging. - -A central supervisor agent (Sentinel Cortex) orchestrates 16+ specialized agents across these tiers, coordinating perception, analysis, and response through Redis Pub/Sub channels. - ---- - -Screenshot from 2026-03-25 20-42-23 - -Screenshot from 2026-03-25 20-42-34 - -Screenshot from 2026-03-25 20-42-40 - -Screenshot from 2026-03-25 20-42-51 - -Screenshot from 2026-03-25 20-42-59 - -Screenshot from 2026-03-25 20-44-39 - -Screenshot from 2026-03-25 20-44-48 - -## Architecture Diagram - -``` -+-----------------------------------------------------------------------------------+ -| SENTINEL AI PLATFORM | -+-----------------------------------------------------------------------------------+ -| | -| +---------------------------+ +---------------------------+ | -| | DATA SOURCES | | EXTERNAL SYSTEMS | | -| | | | | | -| | RTSP/USB/ONVIF Cameras | | PACS (Card Readers) | | -| | MQTT/Modbus Sensors | | Alarm Panels | | -| | Audio Microphones | | SIEM (Splunk/ELK) | | -| | Environmental Monitors | | Building Mgmt Systems | | -| | Body Cameras | | Intercom / PA Systems | | -| +------------+--------------+ +------------+--------------+ | -| | | | -| v v | -| +------------------------------------------------------------------------+ | -| | FASTAPI BACKEND (async) | | -| | | | -| | +-------------------+ +-------------------+ +--------------------+ | | -| | | VIDEO CAPTURE | | YOLO DETECTOR | | GEMINI PROVIDERS | | | -| | | Ring buffers (30f)| | YOLOv8n + ByteTrack| | Flash (realtime) | | | -| | | RTSP reconnection | | 80+ object classes | | Pro (forensics) | | | -| | | Up to 16 cameras | | Dwell/trajectory | | 10 req/min limit | | | -| | +-------------------+ +-------------------+ +--------------------+ | | -| | | | -| | +-------------------+ +-------------------+ +--------------------+ | | -| | | CLIP PIPELINE | | THREAT ENGINE | | ALERT MANAGER | | | -| | | ViT-B-32 embeddings| | 165+ signatures | | Dedup (30s window) | | | -| | | Semantic search | | YOLO+Gemini hybrid| | Auto-escalation | | | -| | | Frame similarity | | Auto-learning | | Status transitions | | | -| | +-------------------+ +-------------------+ +--------------------+ | | -| | | | -| | +------------------------------------------------------------------+ | | -| | | AGENT ORCHESTRATION LAYER | | | -| | | | | | -| | | SUPERVISOR PERCEPTION REASONING | | | -| | | +--------------+ +--------------+ +--------------+ | | | -| | | |Sentinel | |SentinelEye | |ThreatAnalyst | | | | -| | | |Cortex | |PatrolAgent | |Investigator | | | | -| | | | | |CrowdMonitor | |Correlator | | | | -| | | |Orchestrates | |LPRAgent | |ReIDAgent | | | | -| | | |all agents, | |AudioAgent | |GhostTracer | | | | -| | | |resolves | |TamperAgent | |CompanionAgent| | | | -| | | |conflicts, | |PPEAgent | +--------------+ | | | -| | | |shift handoff | |EnvAgent | | | | -| | | +--------------+ |MicroBehavior | ACTION | | | -| | | |AnomalyDetect | +--------------+ | | | -| | | +--------------+ |ResponseAction| | | | -| | | |ReportAgent | | | | -| | | Communication: Redis Pub/Sub |DispatchAgent | | | | -| | | Memory: Short-term (Redis) + |ComplianceAgt | | | | -| | | Long-term (PostgreSQL) +--------------+ | | | -| | +------------------------------------------------------------------+ | | -| | | | -| | +-------------------+ +-------------------+ +--------------------+ | | -| | | REST API (80+) | | WEBSOCKET SERVER | | CELERY WORKERS | | | -| | | Auth, Cameras, | | Multiplexed: | | Escalation checks | | | -| | | Zones, Events, | | - Live frames | | Threat feed poll | | | -| | | Alerts, Cases, | | - Alerts | | Baseline learning | | | -| | | Forensics, Agents | | - Agent activity | | Embedding cleanup | | | -| | | Search, Analytics | | - Metrics | | Video retention | | | -| | | Threat, PACS, LPR | | - Notifications | | Report generation | | | -| | +-------------------+ +-------------------+ +--------------------+ | | -| +------------------------------------------------------------------------+ | -| | | | | -| v v v | -| +-------------------+ +-------------------+ +--------------------+ | -| | POSTGRESQL 16 | | QDRANT | | REDIS 7 | | -| | Users, Cameras, | | sentinel_events | | Session cache | | -| | Zones, Events, | | vehicle_sightings | | Agent memory | | -| | Alerts, Cases, | | entity_appearances| | Pub/Sub channels | | -| | Incidents, Threat | | audio_events | | Rate limiting | | -| | Signatures, Audit | | frame_embeddings | | Celery broker | | -| | Logs, Recordings | | (384-dim vectors) | | | | -| +-------------------+ +-------------------+ +--------------------+ | -| | -| +--------------------------------------------------------+ | -| | NEXT.JS 16 FRONTEND | | -| | | | -| | +---------------+ +---------------+ +------------+ | | -| | | SOC DASHBOARD | | FORENSICS | | AGENT UI | | | -| | | Video Wall | | Investigation | | Fleet Mgmt | | | -| | | Alert Feed | | Case Builder | | Chat | | | -| | | Metrics Bar | | Timeline View | | Cortex | | | -| | | Threat Level | | What-If Panel | | Activity | | | -| | +---------------+ +---------------+ +------------+ | | -| | | | -| | +---------------+ +---------------+ +------------+ | | -| | | DETECTION | | OPERATIONS | | ANALYTICS | | | -| | | LPR / ReID | | Incidents | | Trends | | | -| | | Audio Intel | | Dispatch | | Heatmaps | | | -| | | Behavioral AI | | Patrol Routes | | Facility | | | -| | | Entity Track | | SOPs / BOLO | | Map (GIS) | | | -| | +---------------+ +---------------+ +------------+ | | -| | | | -| | 65 pages | 57+ components | 8 hooks | WebSocket | | -| +--------------------------------------------------------+ | -+-----------------------------------------------------------------------------------+ -``` - -### Data Flow - -``` -Camera Feed --> Video Capture (ring buffer) --> YOLO Detection (per frame) - | | - | v - | Threat Engine (165+ signatures) - | | - +--> CLIP Embedding --> Qdrant Alert Manager (dedup + escalate) - | | - +--> Gemini Analysis (rate-limited) WebSocket --> Frontend - | | - +--> Auto Recorder (5-min chunks) Agent Orchestration (Cortex) - | - Redis Pub/Sub --> All Agents -``` - ---- - -## Technology Stack - -### Backend - -| Component | Technology | Purpose | -|-----------|-----------|---------| -| Web Framework | FastAPI (async) | REST API, WebSocket, middleware | -| Language | Python 3.12 | Async/await throughout | -| ORM | SQLAlchemy 2.0 (asyncpg) | Database models, async queries | -| Database | PostgreSQL 16 | Primary data store | -| Vector Database | Qdrant | Semantic search, frame similarity | -| Cache / Pub-Sub | Redis 7 | Sessions, agent comms, rate limiting | -| Task Queue | Celery | Background jobs, scheduled tasks | -| Object Detection | YOLOv8n + ByteTrack | Real-time detection and tracking | -| Visual Embeddings | CLIP ViT-B-32 | Frame similarity, semantic video search | -| Text Embeddings | all-MiniLM-L6-v2 | Event embedding for search (384-dim) | -| LLM (Primary) | Google Gemini (Flash/Pro) | Scene analysis, forensics, chat | -| LLM (Fallback) | Ollama (gemma3:4b) | Local inference when Gemini unavailable | -| Computer Vision | OpenCV | Frame capture, image processing | -| Migrations | Alembic | Schema versioning | -| Auth | JWT (HS256) + bcrypt | Token auth with role-based access | -| Observability | Prometheus, OpenTelemetry | Metrics, distributed tracing | -| Logging | structlog | Structured JSON logging | - -### Frontend - -| Component | Technology | Purpose | -|-----------|-----------|---------| -| Framework | Next.js 16.1.6 (Turbopack) | App Router, SSR | -| Language | TypeScript + React 19 | Type-safe components | -| Styling | Tailwind CSS v4 | Utility-first dark theme | -| UI Components | Radix UI (10 packages) | Accessible dialog, tabs, toast, etc. | -| Icons | Lucide React | 560+ security-relevant icons | -| Charts | Recharts | Bar, area, radial charts | -| Maps | Leaflet + react-leaflet | Facility maps, geofences, tracking | -| State | React hooks + WebSocket | Real-time reactive updates | - -### Infrastructure - -| Component | Technology | Purpose | -|-----------|-----------|---------| -| Containers | Docker Compose | PostgreSQL, Qdrant, Redis | -| Reverse Proxy | Nginx | Production routing | -| GPU Support | CUDA (optional) | FP16 inference on RTX cards | - ---- - -## Project Structure - -``` -sentinel-ai/ -| -+-- backend/ -| +-- main.py # App entrypoint, lifespan, 80+ router mounts -| +-- config.py # Pydantic settings (DB, AI, video, auth) -| +-- database.py # Async SQLAlchemy engine + session factory -| +-- models/ -| | +-- models.py # Core: User, Camera, Zone, Event, Alert, Case, etc. -| | +-- phase2b_models.py # Incident, Visitor, VideoWall, SIEM, VIP, Lockdown -| | +-- phase3_models.py # Baselines, ContextRule, Intent, AlarmCorrelation -| | +-- agent_state.py # AgentState, AgentMemory persistence -| | +-- agent_audit.py # AgentAuditLog for all agent actions -| +-- schemas/ # Pydantic request/response models -| +-- api/ # 80+ REST endpoint modules -| | +-- auth.py # Login, register, token refresh, RBAC -| | +-- cameras.py # Camera CRUD, health, ONVIF -| | +-- zones.py # Zone polygons, occupancy, breach alerts -| | +-- events.py # Event recording, search, filtering -| | +-- alerts.py # Alert lifecycle, escalation, assignment -| | +-- cases.py # Case management, evidence chain -| | +-- forensics.py # Deep analysis, timeline, correlation -| | +-- agents.py # Agent fleet status, lifecycle, audit -| | +-- copilot.py # Natural language SOC assistant -| | +-- ws.py # WebSocket multiplexing -| | +-- lpr.py # License plate recognition -| | +-- audio.py # Audio event detection -| | +-- weapon_detection.py # Weapon/tool detection -| | +-- pacs.py # Physical access control -| | +-- incidents.py # Incident lifecycle + SLA -| | +-- privacy.py # PII masking, GDPR/CCPA -| | +-- ... (60+ more) -| +-- services/ # 80+ service modules -| | +-- video_capture.py # Threaded multi-camera capture -| | +-- yolo_detector.py # YOLOv8 + ByteTrack detection -| | +-- gemini_provider.py # Gemini API client (rate-limited) -| | +-- gemini_analyzer.py # Real-time scene analysis -| | +-- gemini_forensics.py # Deep forensic frame analysis -| | +-- clip_pipeline.py # Background CLIP embedding pipeline -| | +-- clip_embedder.py # CLIP model (ViT-B-32, 512-dim) -| | +-- vector_store.py # Qdrant client (5 collections) -| | +-- threat_engine.py # 165+ hybrid threat signatures -| | +-- alert_manager.py # Dedup, escalation, correlation -| | +-- baseline_learning.py # Per-zone normality baselines -| | +-- causal_reasoning.py # Incident chain analysis -| | +-- counterfactual.py # What-if scenario engine -| | +-- intent_classifier.py # Behavioral intent analysis -| | +-- autonomous_response.py # Auto threat response -| | +-- ... (60+ more) -| +-- agents/ # 16+ autonomous AI agents -| | +-- base_agent.py # Base with memory, tools, circuit breaker -| | +-- sentinel_cortex.py # Supervisor / orchestrator -| | +-- sentinel_eye.py # Continuous camera monitoring -| | +-- threat_analyst.py # Threat scoring and classification -| | +-- investigator.py # Multi-step case investigation -| | +-- correlator.py # Cross-camera event correlation -| | +-- lpr_agent.py # License plate recognition -| | +-- audio_agent.py # Audio event detection -| | +-- crowd_monitor.py # Crowd density and flow -| | +-- patrol_agent.py # Guard patrol optimization -| | +-- ghost_tracer.py # Predictive entity tracking -| | +-- ... (5+ more) -| +-- middleware/ -| | +-- rate_limit.py # Token bucket (100 req/min default) -| | +-- prometheus_metrics.py # Prometheus export -| | +-- circuit_breaker.py # External API circuit breaker -| | +-- csrf_protection.py # CSRF token validation -| | +-- security_headers.py # HSTS, CSP, X-Frame-Options -| | +-- tracing.py # OpenTelemetry distributed tracing -| +-- tasks/ # Celery periodic tasks -| +-- templates/ # Email/report templates -| +-- reports/ # Generated report output -| -+-- frontend/ -| +-- src/ -| | +-- app/ # 65 page routes (Next.js App Router) -| | | +-- layout.tsx # Root layout, sidebar, auth, copilot -| | | +-- login/ # Authentication page -| | | +-- command-center/ # SOC command center -| | | +-- video-wall/ # Multi-camera video wall -| | | +-- alerts/ # Alert management -| | | +-- incidents/ # Incident lifecycle -| | | +-- forensics/ # Forensic analysis -| | | +-- cases/ # Case management -| | | +-- search/ # Semantic + forensic search -| | | +-- agents/ # Agent fleet management -| | | +-- copilot/ # SOC Copilot chat -| | | +-- lpr/ # License plate reader -| | | +-- audio/ # Audio intelligence -| | | +-- behavioral/ # Behavioral AI -| | | +-- entity-tracking/ # Entity lifecycle -| | | +-- reid/ # Person re-identification -| | | +-- threat-config/ # Threat signature config -| | | +-- pacs/ # Access control -| | | +-- patrol/ # Patrol command -| | | +-- analytics/ # Dashboards and trends -| | | +-- zones/ # Zone management -| | | +-- site-map/ # GIS facility map -| | | +-- floor-plans/ # Floor plan engine -| | | +-- compliance/ # Compliance dashboard -| | | +-- privacy/ # Privacy and GDPR -| | | +-- settings/ # System settings -| | | +-- admin/ # Multi-tenant administration -| | | +-- ... (40+ more) -| | +-- components/ -| | | +-- soc/ # SOC dashboard (9 components) -| | | +-- forensics/ # Investigation tools (12 components) -| | | +-- agents/ # Agent UI (6 components) -| | | +-- common/ # Shared utilities (22 components) -| | | +-- copilot/ # Copilot chat widget -| | | +-- map/ # Leaflet facility map -| | | +-- lpr/ # LPR visualization (4 components) -| | | +-- audio/ # Audio heatmap timeline -| | | +-- threat-intel/ # Threat intel panels (3 components) -| | | +-- ui/ # Base UI (Badge, Card) -| | +-- hooks/ -| | | +-- useWebSocket.ts # Multi-channel WebSocket with reconnect -| | | +-- useLiveFeed.ts # Frame + analysis batching -| | | +-- useAlerts.ts # Alert CRUD + real-time -| | | +-- useThreatResponse.ts # Threat response tracking -| | | +-- useSmartPolling.ts # Visibility-aware polling -| | | +-- useApiError.ts # Error handling wrapper -| | | +-- useDebounce.ts # Debounced state -| | | +-- useKeyboardShortcuts.ts # Keyboard bindings -| | +-- lib/ -| | +-- types.ts # 65+ TypeScript interfaces -| | +-- utils.ts # apiFetch, formatters, API config -| | +-- export.ts # CSV, PDF, JSON export utilities -| +-- package.json -| +-- tailwind config (v4, CSS-based) -| -+-- docker-compose.yml # PostgreSQL 16, Qdrant, Redis -+-- docker-compose.prod.yml # Production with Nginx -+-- nginx.conf # Reverse proxy config -+-- requirements.txt # Python dependencies -+-- alembic.ini # Database migration config -+-- yolov8n.pt # YOLO model weights -+-- yolov8n-pose.pt # Pose estimation model weights -``` - ---- - -## Agent System - -Sentinel AI runs a hierarchical multi-agent system where 16+ specialized agents operate autonomously, coordinated by a central supervisor. - -### Agent Tiers - -**Supervisor Tier** - -| Agent | Role | -|-------|------| -| Sentinel Cortex | Central intelligence coordinator. Orchestrates all agents, resolves conflicting assessments, manages shift handoffs, produces situation reports. | - -**Perception Tier (Always-On)** - -| Agent | Role | -|-------|------| -| SentinelEye | Continuous round-robin camera monitoring with YOLO + Gemini scene analysis | -| PatrolAgent | Guard patrol tracking and route optimization | -| AnomalyDetector | Statistical anomaly detection against learned baselines | -| CrowdMonitor | Crowd density estimation, flow analysis, stampede risk | -| LPRAgent | License plate recognition and database lookup | -| AudioAgent | Audio event classification (gunshots, screams, glass) | -| EnvironmentalAgent | Temperature, humidity, air quality monitoring | -| TamperAgent | Camera tampering detection (blur, tilt, obstruction) | -| PPEAgent | Personal protective equipment compliance | -| MicroBehaviorAgent | Micro-expression and gait anomaly analysis | - -**Reasoning Tier** - -| Agent | Role | -|-------|------| -| ThreatAnalyst | Threat severity scoring and kill-chain classification | -| Investigator | Multi-step autonomous incident investigation | -| Correlator | Cross-camera, cross-modal event correlation | -| ReIDAgent | Person re-identification across cameras | -| GhostTracer | Predictive tracking of disappeared entities | -| CompanionAgent | Companion and group relationship discovery | - -**Action Tier** - -| Agent | Role | -|-------|------| -| ResponseAction | Autonomous threat response (alerts, lockdowns, access revocation) | -| ReportAgent | Automated report generation and formatting | -| DispatchAgent | Operator dispatch, resource assignment | -| ComplianceAgent | Compliance-aware response gating | - -### Agent Infrastructure - -Each agent inherits from a base class that provides: - -- **Async function-calling loop** with Gemini or Ollama -- **Short-term memory** stored in Redis for fast access -- **Long-term memory** persisted to PostgreSQL -- **Circuit breaker** with exponential backoff for failure recovery -- **Token budgeting** per cycle (50,000 tokens default) -- **Tool registry** with access to 50+ specialized tools -- **Full audit logging** of every decision and action -- **Adaptive idle slowdown** to conserve compute when nothing is happening - -### Agent Communication - -Agents communicate through Redis Pub/Sub on dedicated channels: - -``` -sentinel:agents:cortex # Supervisor directives -sentinel:agents:perceptions # Perception observations -sentinel:agents:threats # Threat assessments -sentinel:agents:actions # Action events -sentinel:agents:investigation # Investigation steps -sentinel:agents:anomalies # Anomaly reports -sentinel:agents:correlation # Event correlations -sentinel:agents:predictions # Predictive insights -sentinel:agents:heartbeat # Health monitoring -``` - ---- +**Turn ordinary camera feeds into an autonomous, reasoning command center β€” with local AI, zero cloud, and no data ever leaving your network.** -## Backend Services - -The backend contains 80+ service modules organized by domain. - -### Core Processing - -| Service | Description | -|---------|------------| -| video_capture.py | Threaded multi-camera capture with 30-frame ring buffers, RTSP fallback, auto-reconnection | -| yolo_detector.py | YOLOv8n inference with ByteTrack tracking, dwell time, trajectory analysis | -| clip_pipeline.py | Background CLIP embedding pipeline (ViT-B-32, 512-dim frame vectors) | -| alert_manager.py | Alert deduplication (30s window), severity-based escalation, cross-alert correlation | -| threat_engine.py | 165+ threat signatures with hybrid YOLO + Gemini detection methods | -| auto_recorder.py | Continuous and event-triggered video recording (5-min chunks, 72h retention) | - -### AI and LLM - -| Service | Description | -|---------|------------| -| gemini_provider.py | Gemini API client with built-in rate limiting (10 req/min) | -| gemini_analyzer.py | Real-time scene analysis via Gemini Flash | -| gemini_forensics.py | Deep forensic frame analysis via Gemini Pro | -| ollama_provider.py | Local Ollama model provider as Gemini fallback | -| ai_text_service.py | Text generation and summarization | -| causal_reasoning.py | Causal inference for incident chain analysis | -| counterfactual_engine.py | "What-if" scenario simulation | -| intent_classifier.py | Behavioral intent classification (authorized/evasive/forced_entry) | -| explanation_builder.py | Explainable AI reasoning chain generation | - -### Vector Search - -| Service | Description | -|---------|------------| -| vector_store.py | Qdrant client managing 5 collections: sentinel_events, vehicle_sightings, entity_appearances, audio_events, frame_embeddings | -| clip_embedder.py | CLIP model management for visual similarity search | - -### Operational - -| Service | Description | -|---------|------------| -| incident_lifecycle_service.py | Incident state machine with SLA timers | -| autonomous_response.py | Automated threat response actions | -| pending_action_service.py | Human-in-the-loop approval queue | -| operation_mode.py | Adaptive operation modes (normal/alert/lockdown) | -| emergency_codes.py | No-auth emergency code execution | -| shift_briefing.py | Shift handoff report generation | -| sop_engine.py | Standard Operating Procedure automation | -| patrol_optimizer.py | Guard route optimization | - -### Integration - -| Service | Description | -|---------|------------| -| pacs_service.py | Physical Access Control System (card readers, turnstiles) | -| alarm_panel_service.py | Alarm panel integration | -| alarm_correlation_engine.py | Cross-modal alarm fusion (PACS + video + sensors) | -| siem_service.py | SIEM integration (Splunk, ELK) | -| mqtt_service.py | MQTT sensor data ingestion | -| onvif_service.py | ONVIF camera protocol support | -| bms_service.py | Building Management System integration | -| elevator_service.py | Elevator system integration | -| bodycam_service.py | Body camera integration | -| gis_service.py | Geographic Information System mapping | - -### Privacy and Compliance - -| Service | Description | -|---------|------------| -| privacy_engine_service.py | PII masking, privacy zones, data anonymization | -| compliance_auditor.py | Regulatory audit trail (SOC2, HIPAA, PCI-DSS, GDPR/CCPA) | -| compliance_dashboard_service.py | Compliance metrics and dashboards | +
---- - -## API Surface - -The backend exposes 80+ REST API endpoint groups and a multiplexed WebSocket server. - -### Endpoint Categories - -| Category | Prefix | Endpoints | Description | -|----------|--------|-----------|------------| -| Authentication | /api/auth | 4 | Login, register, token refresh, user management | -| Cameras | /api/cameras | 5 | CRUD, health checks, ONVIF discovery | -| Zones | /api/zones | 5 | Polygon CRUD, occupancy tracking, breach alerts | -| Events | /api/events | 8 | Recording, search, filtering, correlation, export | -| Alerts | /api/alerts | 10 | Lifecycle, escalation, assignment, acknowledge, resolve | -| Cases | /api/cases | 8 | Case CRUD, evidence attachment, AI insights | -| Forensics | /api/forensics | 6 | Deep analysis, timeline, context windows | -| Video | /api/video-* | 12 | Playback, bookmarks, archive, summary, wall layouts | -| Search | /api/search | 3 | Full-text, semantic, visual similarity | -| Agents | /api/agents | 8 | Fleet status, lifecycle, audit, conversation | -| Copilot | /api/copilot | 3 | Natural language SOC assistant | -| LPR | /api/lpr | 4 | Plate detection, database, cross-site correlation | -| Audio | /api/audio | 3 | Audio event detection and classification | -| Threat | /api/threat-* | 12 | Signatures, config, intel feeds, response | -| PACS | /api/pacs | 4 | Access control events, door status | -| Incidents | /api/incidents | 6 | Incident lifecycle with SLA tracking | -| Behavioral | /api/behavioral-* | 4 | Behavioral analytics, baselines, intent | -| Privacy | /api/privacy | 5 | PII masking, privacy zones, GDPR | -| Compliance | /api/compliance | 4 | Audit trails, regulatory dashboards | -| Integration | /api/alarm-panels, iot-sensors, siem, etc. | 18 | External system connectors | -| Emergency | /api/emergency | 3 | No-auth lockdown, evacuation codes | - -### WebSocket Channels - -The `/ws` endpoint multiplexes the following real-time channels: - -| Channel | Data | Rate | -|---------|------|------| -| frame / frames | Live MJPEG frames (base64) | Up to 15 FPS | -| alert / alerts | New and updated alerts | Event-driven | -| metric / metrics | System KPIs | 1/s | -| notifications | User notifications | Event-driven | -| agent_activity | Agent execution logs | Event-driven | -| pending_actions | HITL approval queue | Event-driven | -| analysis | Gemini analysis results | Rate-limited | -| crowd_sentiment | Crowd density/mood data | 1/s | -| companion_discovery | Entity relationship detections | Event-driven | -| ghost_trace | Predictive entity paths | Event-driven | -| threat_response | Active response actions | Event-driven | - ---- - -## Frontend Application - -The frontend is a Next.js 16 application with 65 page routes organized into 9 navigation groups. - -### Page Groups - -**Operations** (7 pages): Dashboard, Command Center, Video Wall, Cameras, System Status, SOC Workspace, SOC Copilot - -**Alerts and Response** (8 pages): Emergency Response, Alerts, Incidents, Threat Response, Alarm Analysis, Pending Actions (HITL), Mass Notifications, Evacuation - -**Investigation** (7 pages): Search (Forensic/Visual/Semantic), Forensics, Cases, Evidence, Link Analysis, Video Summary, Video Archive - -**Detection and AI** (8 pages): Agent Fleet Management, Auto Investigations, Behavioral AI, Context Intelligence, Entity Tracking, Person Re-ID, Plate Reader, Audio Intelligence - -**Threat Management** (6 pages): Threat Config, Signatures, Threat Intel Feeds, Insider Threat, Tamper Detection, BOLO and Logbook - -**Access and Patrol** (7 pages): Access Control (PACS), Visitors, VIP Protection, Patrol Command, Dispatch, Crowd Protocols, SOP Manager - -**Analytics and Maps** (7 pages): Analytics Dashboards, Zones, Site Map (GIS), Floor Plans, Global Overwatch, Environmental Safety, SLA Dashboard - -**Compliance and Privacy** (2 pages): Compliance Dashboard, Privacy and GDPR Center - -**System** (5 pages): Integrations, Webhooks, Settings, Customer Portal, Administration +[![Local First](https://img.shields.io/badge/AI-100%25_Local-06b6d4?style=for-the-badge&logo=ollama&logoColor=white)](#-why-sentigon) +[![No Cloud](https://img.shields.io/badge/Cloud-None-1f2937?style=for-the-badge)](#-why-sentigon) +[![Python](https://img.shields.io/badge/Python-3.12-3776AB?style=for-the-badge&logo=python&logoColor=white)](#-tech-stack) +[![Next.js](https://img.shields.io/badge/Next.js-16-000000?style=for-the-badge&logo=nextdotjs&logoColor=white)](#-tech-stack) +[![FastAPI](https://img.shields.io/badge/FastAPI-async-009688?style=for-the-badge&logo=fastapi&logoColor=white)](#-tech-stack) -### Key Component Libraries +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-22c55e?style=flat-square)](#-contributing) +![Stars](https://img.shields.io/github/stars/Sherin-SEF-AI/Sentigon?style=flat-square&color=eab308) +![Last commit](https://img.shields.io/github/last-commit/Sherin-SEF-AI/Sentigon?style=flat-square&color=06b6d4) +![Issues](https://img.shields.io/github/issues/Sherin-SEF-AI/Sentigon?style=flat-square) -**SOC Components** (9 files, 2,000+ lines): Video wall with configurable grid layouts, live alert feed with status colors, camera grid, crowd sentiment badges, live frame/analysis panel, metrics bar, threat level indicator, zone overlays. +
-**Forensics Components** (12 files, 130,000+ lines): Agent decision timeline, case builder, detection overlay, incident list, incident replay player, investigation panel, multi-camera replay grid, reconstruction panel, semantic search bar, similar frames panel, timeline view, what-if scenario panel. +**[ Why ](#-why-sentigon)** Β· **[ Features ](#-what-it-actually-does)** Β· **[ The Console ](#%EF%B8%8F-the-command-center)** Β· **[ Quick Start ](#-quick-start-5-minutes)** Β· **[ Architecture ](#%EF%B8%8F-architecture)** Β· **[ Tech ](#-tech-stack)** -**Agent Components** (6 files, 75,000+ lines): Agent activity panel, chat interface with Sentinel Cortex, control panel, conversation history, status grid by tier, Cortex overview dashboard. - -**Common Components** (22 files): Annotation canvas, bounding box overlay, confidence slider, connection status, data state (loading/empty/error), error boundary, file upload, metric sparklines, neural glass overlay, pose overlay, skeleton loaders, SLA countdown, system health gauge, timeline view, toast notifications. - -### Design System - -The frontend uses a dark theme optimized for SOC environments: - -- Background: #030712 (near-black) -- Card surfaces: #111827 -- Accent color: #06b6d4 (cyan) -- Severity mapping: red (critical), orange (high), yellow (medium), blue (low) -- Custom animations: pulse-alert, threat-pulse, slide-in, fade-in, rec-blink -- Glow effects for severity indicators (.glow-red, .glow-cyan, etc.) +
--- -## Database Schema - -### Core Models - -| Model | Key Fields | Purpose | -|-------|-----------|---------| -| User | email, hashed_password, role, is_active | Authentication, RBAC | -| Camera | name, source, location, status, fps, zone_id, config (JSONB) | Camera registry | -| Zone | name, zone_type, polygon, max_occupancy, alert_on_breach | Geographic zones | -| Event | camera_id, zone_id, event_type, severity, confidence, detections (JSONB), gemini_analysis | Detection events | -| Alert | event_id, severity, status, threat_type, assigned_to, correlated_alert_ids | Alert lifecycle | -| Case | title, status, priority, assigned_to, ai_insights | Investigation cases | -| CaseEvidence | case_id, evidence_type, reference_id, file_url | Evidence chain | -| Recording | camera_id, file_path, duration_seconds, event_id | Video recordings | -| VideoBookmark | recording_id, timestamp_offset, label, ai_analysis | Bookmarked moments | -| ThreatSignature | name, category, severity, detection_method, yolo_classes, gemini_prompt | Threat rules | -| AuditLog | user_id, action, resource_type, resource_id, ip_address | Audit trail | -| ActivityNotification | name, rule_nl, severity, zones, schedule | NL alert rules | - -### Extended Models - -| Model | Purpose | -|-------|---------| -| Incident | Full incident lifecycle with SLA timers (acknowledge/respond/resolve) | -| IncidentStatusLog | Status change audit trail | -| Visitor | Pre-registration, check-in/out, overstay detection | -| NotificationBroadcast | Multi-channel emergency broadcasts (email, SMS, push, PA, signage) | -| VideoWallLayout | Multi-monitor grid configuration | -| BehavioralEvent | Suspicious pattern detection (loitering, casing, tailgating) | -| SIEMConnector | External SIEM integration configuration | -| VIPProfile | VIP tracking with location alerts | -| SecurityProcedure | SOP management with automation triggers | -| LockdownSequence | Emergency lockdown automation | - -### Intelligence Models - -| Model | Purpose | -|-------|---------| -| ActivityBaseline | Learned per-camera/zone/time-slot normality profiles | -| ContextRule | Zone-type-aware context scoring (restricted/kitchen/lobby/server_room) | -| IntentClassification | Behavioral intent (authorized_access/reconnaissance/evasive/forced_entry) | -| AlertFeedback | User feedback for ML model tuning | -| AlarmCorrelation | Cross-modal alarm fusion (PACS + video + sensors) | -| EntityAppearance | Re-identification descriptors for person tracking | -| WeaponDetection | Weapon/tool detection state management | - -### Vector Collections (Qdrant) - -| Collection | Dimensions | Content | -|------------|-----------|---------| -| sentinel_events | 384 | Text embeddings of security events | -| vehicle_sightings | 384 | Vehicle description embeddings | -| entity_appearances | 384 | Person appearance embeddings | -| audio_events | 384 | Audio event description embeddings | -| frame_embeddings | 512 | CLIP visual frame embeddings | +> **TL;DR** β€” Sentigon ingests live camera feeds and runs them through a fleet of **12 cooperating AI agents** (perception β†’ reasoning β†’ action β†’ supervision) that *detect, verify, reason about, and respond to* physical-security threats in real time. Every model β€” vision, language, OCR, audio β€” runs **locally**. There is **no cloud, no API key, no telemetry**. Your footage stays yours. ---- +
-## Threat Detection Engine +## πŸ€” Why Sentigon? -The platform ships with 165+ threat signatures using a hybrid detection approach. +Most "AI security" products are a thin wrapper around a cloud API β€” your footage leaves the building, you pay per frame, and a "person detected" box is the extent of the intelligence. Sentigon is the opposite: -### Detection Methods +| | Typical AI surveillance | **Sentigon** | +|---|---|---| +| **Where AI runs** | Someone else's cloud | 🏠 **Your hardware, fully offline** | +| **Your footage** | Uploaded & retained | πŸ”’ **Never leaves your network** | +| **Cost model** | Per-frame / per-camera SaaS | πŸ’Έ **$0 β€” open & self-hosted** | +| **Intelligence** | Bounding boxes | 🧠 **Agents that reason, verify, and act** | +| **False positives** | You drown in them | βœ… **An adversarial verifier kills them** | +| **Lock-in** | Proprietary | πŸ”“ **Open, hackable, yours** | -**YOLO-based**: Triggers when specific object classes are detected (person in restricted zone, knife, backpack in sterile area). +
-**Gemini-based**: Triggers when Gemini scene analysis contains specific keywords or matches a structured prompt. +## πŸš€ What it actually does -**Hybrid**: Combines YOLO object detection with Gemini contextual analysis for higher confidence. +A single self-hosted platform that replaces a rack of disconnected tools: -### Signature Categories +### πŸ‘οΈ Perception β€” *see everything* +- **State-of-the-art detection** β€” RT-DETR (transformer, NMS-free), YOLO11, **YOLO-World open-vocabulary** ("knife", "person on the ground", "fire" by text prompt β€” no retraining), BoT-SORT re-ID tracking, **SAM2** mask segmentation for occlusion. +- **Structured scene intelligence** β€” a local vision model (qwen2.5-VL) produces a real **scene graph** (objects, attributes, relationships), captions, activities, and an *evidence-calibrated* threat assessment. +- **Behavior over time** β€” geometric, hallucination-free temporal detection of **loitering, running, falls** (pose-based), and **abandoned objects** β€” behaviors that only exist across frames. +- **Real ALPR** (local EasyOCR plate reading) Β· **audio event detection** (gunshot / glass-break / scream / alarm) Β· **CLIP** appearance embeddings. -| Category | Examples | -|----------|---------| -| Intrusion and Access | Unauthorized entry, tailgating, perimeter breach, fence climbing | -| Violence and Weapons | Weapon detected, physical altercation, aggressive posture | -| Suspicious Behavior | Loitering, casing, abandoned objects, unusual movement patterns | -| Vehicle Anomalies | Restricted zone entry, wrong-way driving, speeding, abandoned vehicle | -| Safety Hazards | Fire/smoke, person down, medical emergency, fall detection | -| Compliance Violations | PPE missing, smoking in restricted area, hard hat violation | -| Crowd Events | Overcrowding, stampede risk, unauthorized gathering | -| Cyber-Physical | Camera tampering, sensor manipulation, network intrusion indicators | +### 🧠 Reasoning β€” *connect the dots* +- **Adversarial threat verifier** β€” a second, skeptical AI re-examines every flagged threat and tries to *refute* it. Only the survivors become alerts. This is the single biggest lever against alert fatigue. +- **Escalation chains** β€” recognizes a *sequence* on one person (loiter β†’ test door β†’ approach) and escalates it long before any single step would. +- **Trajectory prediction**, **cross-camera entity tracking & re-ID**, **real-time BOLO** appearance/plate matching, **semantic "looks-like" forensic search**. +- **Adaptive thresholds** that learn each camera's normal and stop crying wolf in naturally-busy areas. -### Auto-Learning +### ⚑ Action β€” *do something about it* +- **Autonomous response** pipeline (incident recording β†’ SOP playbook β†’ operator dispatch β†’ emergency-services lookup) with shadow-mode safety. +- **SOC Copilot** β€” an agentic, tool-using chat that answers *"what's happening on the loading dock right now?"* with a reasoned, data-grounded answer. +- **SOP execution**, **compliance forecasting**, **predictive analytics**, **red-team self-testing**. -The threat engine learns from operator feedback. When an operator marks a detection as a false positive or confirms a true positive, the system adjusts detection thresholds and can generate new signatures from confirmed incidents. +### πŸ”­ Supervision +- A **SENTINEL Cortex** agent orchestrates the fleet, maintains the security posture, issues directives, and synthesizes shift briefings. ---- +
-## Integration Layer +## πŸ–₯️ The Command Center -Sentinel AI connects to physical security infrastructure through dedicated service modules. +A purpose-built **mission-control** interface β€” not a generic admin dashboard. Deep layered surfaces, monospaced telemetry, glowing status LEDs, a live command bar, and real-time agent feeds across **60+ operational views**. -| System | Protocol | Capabilities | -|--------|----------|-------------| -| PACS (Access Control) | API | Card reader events, door lock/unlock, access logs | -| Alarm Panels | API | Zone arm/disarm, alarm trigger/clear, panel status | -| ONVIF Cameras | ONVIF | Discovery, PTZ control, stream URI negotiation | -| IoT Sensors | MQTT, Modbus | Temperature, humidity, air quality, motion | -| SIEM | Syslog, API | Event forwarding to Splunk, ELK Stack | -| Building Management | BACnet, API | HVAC, lighting, elevator control | -| Intercom Systems | SIP, API | Two-way audio, PA announcements | -| Video Management | RTSP, API | Stream ingestion from third-party VMS | -| Body Cameras | API | Footage ingestion and correlation | -| Emergency Services | Webhook | Automated dispatch notifications | + ---- +> _Spin it up (below) and open `http://localhost:3000` to see it live._ -## Quick Start +
-### Prerequisites +## ⚑ Quick Start (5 minutes) -- Python 3.12+ -- Node.js 20+ -- Docker and Docker Compose -- (Optional) NVIDIA GPU with CUDA for accelerated inference +Sentigon runs on **bare metal** β€” no Docker required. Everything (Postgres, Redis, Qdrant, the backend, and the frontend) comes up with one script. -### 1. Start Infrastructure +**Prerequisites:** Python 3.12 Β· Node 20+ Β· [Ollama](https://ollama.com) Β· (a GPU is recommended but not required) ```bash -cd sentinel-ai -docker-compose up -d -``` +# 1. Clone +git clone https://github.com/Sherin-SEF-AI/Sentigon.git +cd Sentigon -This starts PostgreSQL 16 (port 5432), Qdrant (port 6333), and Redis (port 6379). +# 2. Pull the local models (the only "download" you need β€” no API keys, ever) +ollama pull qwen2.5:7b # reasoning / language +ollama pull qwen2.5vl:7b # vision -### 2. Backend Setup - -```bash -# Create and activate virtual environment -python -m venv venv -source venv/bin/activate - -# Install dependencies +# 3. Backend deps (into a venv) +python3.12 -m venv .venv && source .venv/bin/activate pip install -r requirements.txt -# Copy environment config -cp .env.example .env -# Edit .env and set your GEMINI_API_KEY +# 4. Frontend deps +cd frontend && npm install && cd .. -# Start the backend -uvicorn backend.main:app --reload --host 0.0.0.0 --port 8000 +# 5. Launch the entire stack +bash start-local.sh ``` -On startup, the backend will automatically: -- Run database migrations (Alembic) -- Create all tables -- Initialize Qdrant vector collections -- Sync 165+ threat signatures -- Seed the default admin user -- Load YOLOv8 model weights -- Register and start available cameras -- Launch the multi-agent system -- Start background tasks (CLIP pipeline, auto-recorder, monitoring) - -### 3. Frontend Setup - -```bash -cd frontend -npm install -npm run dev -``` - -The frontend will be available at http://localhost:3000. - -### 4. Default Login +Then open **`http://localhost:3000`** and log in: ``` Email: admin@sentinel.local -Password: changeme123 +Password: changeme123 ← change this before exposing it anywhere ``` ---- - -## Configuration - -All configuration is managed through environment variables. See `.env.example` for the complete list. - -### Required - -| Variable | Description | -|----------|------------| -| GEMINI_API_KEY | Google Gemini API key for AI analysis | -| DATABASE_URL | PostgreSQL async connection string | -| SECRET_KEY | JWT signing secret (change in production) | - -### AI and Detection - -| Variable | Default | Description | -|----------|---------|------------| -| GEMINI_RATE_LIMIT | 10 | Gemini requests per minute | -| YOLO_MODEL_PATH | models/yolov8n.pt | YOLO model weights | -| YOLO_CONFIDENCE | 0.45 | Detection confidence threshold | -| CLIP_MODEL | ViT-B-32 | CLIP model variant | -| CLIP_INTERVAL | 3.0 | Seconds between CLIP embeddings | -| LLM_MAX_CONCURRENT | 2 | Max concurrent LLM calls | - -### Video - -| Variable | Default | Description | -|----------|---------|------------| -| MAX_CAMERAS | 16 | Maximum simultaneous cameras | -| CAMERA_BUFFER_SIZE | 30 | Ring buffer frames per camera | -| DEFAULT_FPS | 15 | Target frames per second | -| RECORDING_CHUNK_SECONDS | 300 | Auto-recording chunk duration | -| RECORDING_RETENTION_HOURS | 72 | Recording retention period | - -### Authentication - -| Variable | Default | Description | -|----------|---------|------------| -| ACCESS_TOKEN_EXPIRE_MINUTES | 480 | JWT token expiry | -| DEFAULT_ADMIN_EMAIL | admin@sentinel.local | Seed admin email | -| DEFAULT_ADMIN_PASSWORD | changeme123 | Seed admin password | - -### Infrastructure - -| Variable | Default | Description | -|----------|---------|------------| -| REDIS_URL | redis://localhost:6379/0 | Redis connection | -| QDRANT_HOST | localhost | Qdrant host | -| QDRANT_PORT | 6333 | Qdrant REST port | - ---- - -## RBAC and Authentication +> The backend serves on `:8002`. On first boot it auto-runs migrations, seeds 165+ threat signatures, loads the detector + agents, and registers any available cameras. Add an RTSP/USB camera from **Settings β†’ Cameras**. + +
+ +## πŸ—οΈ Architecture + +
+Sentigon architecture +
+ +A layered **perception β†’ reasoning β†’ action β†’ supervision** pipeline: + +```mermaid +flowchart LR + subgraph Ingest["πŸ“‘ Ingest"] + CAM["RTSP / USB / ONVIF"] + IOT["IoT Β· PACS Β· Alarms"] + end + subgraph Perceive["πŸ‘οΈ Perception"] + DET["RT-DETR / YOLO-World
pose Β· BoT-SORT Β· SAM2"] + VLM["Scene Intelligence
(qwen2.5-VL)"] + AUD["Audio · ALPR · CLIP"] + end + subgraph Reason["🧠 Reasoning"] + VERIFY["Adversarial Verifier"] + TEMP["Temporal · Escalation
Trajectory Β· BOLO"] + CORTEX["SENTINEL Cortex
(orchestrator)"] + end + subgraph Act["⚑ Action"] + RESP["Autonomous Response
SOP Β· Dispatch"] + COPILOT["SOC Copilot"] + end + Ingest --> Perceive --> Reason --> Act + Reason <--> CORTEX +``` -The platform implements role-based access control with four tiers. +Every box runs **locally**. The LLMs are Ollama (`qwen2.5` / `qwen2.5-VL`); the detectors are ultralytics (RT-DETR / YOLO / SAM2); embeddings are CLIP. No external inference calls. -| Role | Level | Capabilities | -|------|-------|-------------| -| Admin | 4 | Full access. User management, system settings, agent control, all operations. | -| Analyst | 3 | Investigations, cases, forensics, reports, search, threat configuration. | -| Operator | 2 | Camera and zone management, alert handling, dispatch, patrol. | -| Viewer | 1 | Dashboard viewing, search, read-only access to events and alerts. | +
-Authentication uses JWT tokens (HS256) with bcrypt password hashing. Tokens expire after 480 minutes by default. All mutation endpoints (POST, PUT, PATCH, DELETE) are logged to the audit trail with user ID, IP address, and timestamp. +## 🧩 Tech Stack -Emergency endpoints (/api/emergency) bypass authentication to allow lockdown and evacuation activation without credentials. +**Backend** β€” FastAPI (async) Β· SQLAlchemy 2.0 + asyncpg Β· PostgreSQL Β· Qdrant (vectors) Β· Redis Β· Celery Β· Alembic Β· JWT/bcrypt Β· Prometheus + structlog +**AI / CV** β€” Ollama (qwen2.5 / qwen2.5-VL) Β· ultralytics (RT-DETR, YOLO11, YOLO-World, pose, SAM2, BoT-SORT) Β· CLIP Β· EasyOCR Β· librosa Β· OpenCV +**Frontend** β€” Next.js 16 (App Router) Β· React 19 Β· TypeScript Β· Tailwind CSS v4 Β· Radix UI Β· Recharts Β· Leaflet ---- +
+πŸ“‚ Deep dive β€” agents, services, and the full feature set (click to expand) -## Deployment +
-### Development +### The agent fleet (12) +**Perception:** Watcher Β· Detector Β· Audio Sentinel β€” **Reasoning:** Threat Analyzer Β· Tracker Β· Investigator β€” **Action:** Responder Β· Reporter β€” **Supervision:** SENTINEL Cortex β€” **Specialized:** Access Guardian Β· Environmental Β· Red Team. -```bash -docker-compose up -d # Infrastructure -uvicorn backend.main:app --reload # Backend with hot reload -cd frontend && npm run dev # Frontend with Turbopack -``` +Agents communicate over Redis pub/sub channels and call internal tools through a local LLM function-calling loop. -### Production +### Notable services +`scene_intelligence` Β· `threat_verifier` Β· `temporal_behavior` Β· `escalation_tracker` Β· `trajectory_predictor` Β· `sam_segmenter` Β· `alpr_service` Β· `audio_detection_service` Β· `bolo_matcher` Β· `adaptive_thresholds` Β· `baseline_learning` Β· `autonomous_response` Β· `sop_engine` Β· `compliance` (with forecasting) Β· `forensic_search` (semantic) Β· `feedback_tuning` Β· `entity_tracker` Β· `context_fusion`. -```bash -docker-compose -f docker-compose.prod.yml up -d -``` +### Surface area +- **60+ frontend views** across Operations, Alerts & Response, Investigation, Detection & AI, Threat Management, Access & Patrol, Analytics & Maps, Compliance, and System. +- **Hundreds of API endpoints**, WebSocket live feeds, RBAC (admin / analyst / operator / viewer), audit logging, and multi-tenant scaffolding. -The production compose file includes Nginx as a reverse proxy with: +
-- Frontend static files served directly -- Backend API proxied to uvicorn workers -- WebSocket upgrade handling -- SSL termination (configure certificates in nginx.conf) +
-### GPU Acceleration +## πŸ—ΊοΈ Roadmap -If an NVIDIA GPU is available, the platform automatically detects CUDA and runs YOLO inference with FP16 half-precision. No configuration changes are required. CPU fallback is automatic. +- [ ] One-command installer & prebuilt model bundle +- [ ] Live multi-camera demo dataset +- [ ] Deep audio model (PANNs / YAMNet) drop-in to replace the DSP classifier +- [ ] Mask-based occlusion **re-acquisition** (SAM2 video memory) +- [ ] Edge deployment guide (Jetson / mini-PC) -### Health Monitoring +
-The backend exposes Prometheus metrics at the standard metrics endpoint. Key metrics include: +## 🀝 Contributing -- HTTP request latency (by endpoint, method, status) -- Active WebSocket connections -- Agent cycle times and error rates -- Camera health scores -- Alert processing latency -- Gemini API response times +Issues, ideas, and PRs are very welcome β€” a new detector, an agent skill, a UI polish pass, or docs. Open an issue to start a conversation. -OpenTelemetry distributed tracing is available for end-to-end request correlation across services. +
---- +### If a private, local-first, agentic SOC is something the world should have β€” -## Middleware Stack +# ⭐ Star the repo -| Middleware | Purpose | -|-----------|---------| -| Rate Limiting | Token bucket: 100 req/min general, 10 req/min login, 30 req/min AI endpoints | -| Prometheus | HTTP latency histograms, business KPI counters | -| Circuit Breaker | Protects external API calls (Gemini, SIEM, PACS) with exponential backoff | -| CSRF Protection | Token validation on state-changing requests | -| Security Headers | HSTS, Content-Security-Policy, X-Frame-Options, X-Content-Type-Options | -| Distributed Tracing | OpenTelemetry trace propagation | -| Audit Logging | All POST/PUT/PATCH/DELETE requests logged with user context | +It genuinely helps, and it's the fastest way to follow where this goes. ---- - ---- - -## Author - -**Sherin Joseph Roy** -[https://sherinjosephroy.link](https://sherinjosephroy.link) - ---- +
-## License +**Built for operators who refuse to send their footage to someone else's cloud.** -Proprietary. All rights reserved. +
diff --git a/backend/agents/consolidated/detector.py b/backend/agents/consolidated/detector.py index e5888c8..7387c0c 100644 --- a/backend/agents/consolidated/detector.py +++ b/backend/agents/consolidated/detector.py @@ -165,27 +165,52 @@ async def _pipeline_lpr(self, camera_id: str, cam: dict, objects: list[dict]) -> if not vehicles: return None cam_name = cam.get("name", camera_id) - result = await self.execute_tool_loop( - prompt=( - f"Analyze current frame from camera {camera_id} ({cam_name}). " - f"{len(vehicles)} vehicle(s) detected. Use analyze_frame_with_gemini to " - f"read all visible license plates. For each plate report: plate_text, " - f"vehicle_type, vehicle_color, confidence. Then use store_observation to " - f"record each reading with category='plate_read'." - ), - context_data={"camera_id": camera_id, "vehicle_count": len(vehicles), "task": "lpr"}, - ) - resp = result.get("response", "") + + # Real local ALPR (EasyOCR) β€” fast, deterministic plate reads instead of + # the slow/inconsistent vision-model path. Each vehicle box is OCR'd and + # checked against active vehicle BOLOs. + from backend.services.alpr_service import alpr_service + if not alpr_service.available(): + return None + from backend.services.video_capture import capture_manager + stream = capture_manager.get_stream(camera_id) + latest = stream.get_latest_frame() if stream else None + if latest is None: + return None + _, frame = latest + + reads: list[dict] = [] + bolo_hits: list[dict] = [] + for v in vehicles: + bbox = v.get("bbox") + if not bbox: + continue + r = await alpr_service.read_and_match(frame, bbox) + if r.get("plate"): + reads.append({"plate": r["plate"], "confidence": r.get("confidence"), "track_id": v.get("track_id")}) + if r.get("matches"): + bolo_hits.append({"plate": r["plate"], "matches": r["matches"], "track_id": v.get("track_id")}) + if not reads: + return None + await self.send_message(CH_PERCEPTIONS, { "type": "plate_read", "camera_id": camera_id, "camera_name": cam_name, - "vehicle_count": len(vehicles), "analysis": resp[:500], + "plates": reads, "bolo_hits": bolo_hits, "timestamp": datetime.now(timezone.utc).isoformat(), }) + for hit in bolo_hits: + await self.send_message(CH_PERCEPTIONS, { + "type": "bolo_vehicle_match", "camera_id": camera_id, "camera_name": cam_name, + "plate": hit["plate"], "severity": "high", + "reason": "; ".join(m.get("reason", "BOLO vehicle") for m in hit["matches"]), + "timestamp": datetime.now(timezone.utc).isoformat(), + }) await self.log_action("lpr_scan", { "camera_id": camera_id, "vehicles_detected": len(vehicles), - "decision": f"LPR scan {cam_name}: {len(vehicles)} vehicle(s)", + "decision": f"ALPR {cam_name}: read {len(reads)} plate(s), {len(bolo_hits)} BOLO hit(s)", }) - return {"vehicles_scanned": len(vehicles), "response": resp[:300]} + return {"vehicles_scanned": len(vehicles), "plates_read": len(reads), + "bolo_hits": len(bolo_hits), "plates": [r["plate"] for r in reads]} # ── Pipeline: PPE Compliance ────────────────────────────────── diff --git a/backend/agents/monitoring_agent.py b/backend/agents/monitoring_agent.py index e3bbcd6..e9943e9 100644 --- a/backend/agents/monitoring_agent.py +++ b/backend/agents/monitoring_agent.py @@ -46,6 +46,8 @@ # AI vision is expensive; only invoke it every N frames per camera. _AI_EVERY_N_FRAMES = 15 +# Refresh learned adaptive thresholds from the baseline service this often. +_THRESHOLD_REFRESH_EVERY_N = 300 # Minimum seconds between full pipeline runs for a single camera. _MIN_INTERVAL_SECONDS = 1.0 @@ -101,6 +103,19 @@ async def process_frame( counter = self._frame_counters.get(camera_id, 0) + 1 self._frame_counters[camera_id] = counter + # Refresh learned adaptive thresholds + active BOLOs into sync caches (throttled). + if counter % _THRESHOLD_REFRESH_EVERY_N == 1: + try: + from backend.services.adaptive_thresholds import adaptive_thresholds + zid = zone_info.get("id") if zone_info else None + async with async_session() as _db: + await adaptive_thresholds.refresh(_db, camera_id, zid) + if getattr(settings, "BOLO_REALTIME_ENABLED", True): + from backend.services.bolo_matcher import bolo_matcher + await bolo_matcher.refresh(_db) + except Exception as exc: # noqa: BLE001 + logger.debug("threshold/bolo refresh failed for %s: %s", camera_id, exc) + if settings.VISION_VERIFIED_DETECTION: # Verified-vision path: structured scene intelligence + an adversarial # verifier. Hallucination-resistant β€” only threats a skeptic confirms @@ -145,6 +160,25 @@ async def process_frame( except Exception as exc: # noqa: BLE001 logger.debug("pose behaviours failed for %s: %s", camera_id, exc) + # ── 3a-quater. Real-time BOLO appearance matching ──────── + # Embed each new person track once and match against active person BOLOs. + if getattr(settings, "BOLO_REALTIME_ENABLED", True): + try: + from backend.services.bolo_matcher import bolo_matcher + if bolo_matcher.has_active(): + threats.extend(bolo_matcher.scan_frame(frame, detections, camera_id)) + except Exception as exc: # noqa: BLE001 + logger.debug("BOLO scan failed for %s: %s", camera_id, exc) + + # ── 3a-quinquies. Threat-escalation chains ─────────────── + # Detect escalating behaviour SEQUENCES on a single entity (e.g. + # loitering β†’ running β†’ fall) across the track-bearing threats above. + try: + from backend.services.escalation_tracker import escalation_tracker + threats.extend(escalation_tracker.observe(camera_id, threats, timestamp_epoch)) + except Exception as exc: # noqa: BLE001 + logger.debug("escalation tracking failed for %s: %s", camera_id, exc) + # ── 3a-ter. SAM2 mask enrichment for FLAGGED objects ────── # Only objects referenced by a threat get a pixel-precise SAM2 mask # (occlusion-robust extent), attached to the threat + its detection so it diff --git a/backend/api/bolo.py b/backend/api/bolo.py index 66d2fd3..cb5ce81 100644 --- a/backend/api/bolo.py +++ b/backend/api/bolo.py @@ -200,6 +200,41 @@ async def deactivate_bolo( raise HTTPException(status_code=500, detail="Failed to deactivate BOLO entry") +class EnrollAppearanceRequest(BaseModel): + image_base64: str = Field(..., description="Reference image (base64 JPEG/PNG)") + bbox: Optional[List[float]] = Field(None, description="[x1,y1,x2,y2]; whole image if omitted") + + +@router.post("/{bolo_id}/enroll-appearance") +async def enroll_appearance(bolo_id: uuid.UUID, body: EnrollAppearanceRequest, _user=Depends(get_current_user)): + """Enroll a person BOLO with a CLIP appearance embedding from a reference + image so the real-time matcher can flag this person across cameras.""" + import base64 + import cv2 + import numpy as np + from backend.services.appearance_embedder import appearance_embedding + from backend.services.bolo_service import bolo_service + + try: + raw = base64.b64decode(body.image_base64.split(",", 1)[-1]) + frame = cv2.imdecode(np.frombuffer(raw, np.uint8), cv2.IMREAD_COLOR) + except Exception: + raise HTTPException(status_code=400, detail="Invalid image_base64") + if frame is None: + raise HTTPException(status_code=400, detail="Could not decode image") + + h, w = frame.shape[:2] + bbox = body.bbox or [0, 0, w, h] + emb = appearance_embedding(frame, bbox) + if not emb: + raise HTTPException(status_code=422, detail="Could not compute appearance embedding") + + ok = await bolo_service.enroll_appearance(bolo_id, emb) + if not ok: + raise HTTPException(status_code=404, detail="BOLO not found") + return {"enrolled": True, "bolo_id": str(bolo_id), "embedding_dim": len(emb)} + + @router.get("/{bolo_id}/sightings", response_model=List[dict]) async def get_bolo_sightings( bolo_id: uuid.UUID, diff --git a/backend/api/compliance_dashboard.py b/backend/api/compliance_dashboard.py index 43ccb00..48c07af 100644 --- a/backend/api/compliance_dashboard.py +++ b/backend/api/compliance_dashboard.py @@ -53,6 +53,20 @@ async def compliance_history( raise HTTPException(400, str(e)) +@router.get("/forecast") +async def compliance_forecast( + framework: str = "gdpr", + target: float = Query(0.8, ge=0.0, le=1.0), + db: AsyncSession = Depends(get_db), +): + """Project the compliance score from its recent trend and estimate when it + will breach the target β€” enabling proactive remediation.""" + try: + return await compliance_dashboard_service.forecast_compliance(db, framework=framework, target=target) + except Exception as e: + raise HTTPException(400, str(e)) + + @router.get("/issues") async def compliance_issues( severity: str = None, diff --git a/backend/api/forensics.py b/backend/api/forensics.py index f4e7d39..c8b56d5 100644 --- a/backend/api/forensics.py +++ b/backend/api/forensics.py @@ -485,6 +485,40 @@ async def subject_search( } +class ImageSearchRequest(BaseModel): + image_base64: str = Field(..., description="Query image (base64 JPEG/PNG)") + bbox: Optional[List[float]] = Field(None, description="[x1,y1,x2,y2] crop; whole image if omitted") + camera_id: Optional[str] = None + max_results: int = Field(20, ge=1, le=100) + + +@router.post("/search-by-image") +async def search_by_image( + body: ImageSearchRequest, + _user=Depends(require_role(UserRole.ANALYST)), +): + """Image-based "looks-like" forensic search: upload an image (or a crop) and + find the most visually similar people/objects seen across all cameras, using + CLIP embeddings against the object-crop vector store. + """ + import base64 + import cv2 + import numpy as np + from backend.services.forensic_search_service import forensic_search_service + + try: + raw = base64.b64decode(body.image_base64.split(",", 1)[-1]) + frame = cv2.imdecode(np.frombuffer(raw, np.uint8), cv2.IMREAD_COLOR) + except Exception: + raise HTTPException(status_code=400, detail="Invalid image_base64") + if frame is None: + raise HTTPException(status_code=400, detail="Could not decode image") + + return await forensic_search_service.search_objects_by_image( + frame, bbox=body.bbox, camera_id=body.camera_id, top_k=body.max_results, + ) + + @router.post("/movement-trail") async def movement_trail( body: MovementTrailRequest, diff --git a/backend/api/intelligence.py b/backend/api/intelligence.py index b7aa386..9af1fce 100644 --- a/backend/api/intelligence.py +++ b/backend/api/intelligence.py @@ -160,6 +160,50 @@ async def deep_analyze(body: SceneAnalyzeRequest): } +class AnalyzeAudioRequest(BaseModel): + audio_base64: str = Field(..., description="WAV audio (base64)") + + +@router.post("/analyze-audio") +async def analyze_audio(body: AnalyzeAudioRequest): + """Detect security-relevant sound events (gunshot/glass/scream/alarm) in a WAV + clip via the local audio DSP engine.""" + import base64 + from backend.services.audio_detection_service import audio_detection_service + try: + raw = base64.b64decode(body.audio_base64.split(",", 1)[-1]) + except Exception: + raise HTTPException(status_code=400, detail="Invalid audio_base64") + events = audio_detection_service.analyze_bytes(raw) + return {"event_count": len(events), "events": events} + + +class ReadPlateRequest(BaseModel): + image_base64: str = Field(..., description="Image containing a plate (base64)") + bbox: Optional[List[float]] = Field(None, description="Optional vehicle/plate crop [x1,y1,x2,y2]") + + +@router.post("/read-plate") +async def read_plate(body: ReadPlateRequest): + """Read a license plate via the local ALPR (EasyOCR) engine and check it + against active vehicle BOLOs.""" + import base64 + import cv2 + import numpy as np + from backend.services.alpr_service import alpr_service + + try: + raw = base64.b64decode(body.image_base64.split(",", 1)[-1]) + frame = cv2.imdecode(np.frombuffer(raw, np.uint8), cv2.IMREAD_COLOR) + except Exception: + raise HTTPException(status_code=400, detail="Invalid image_base64") + if frame is None: + raise HTTPException(status_code=400, detail="Could not decode image") + if not alpr_service.available(): + raise HTTPException(status_code=503, detail="ALPR engine unavailable") + return await alpr_service.read_and_match(frame, body.bbox) + + class SegmentRequest(BaseModel): camera_id: Optional[str] = None image_base64: Optional[str] = None diff --git a/backend/config.py b/backend/config.py index da64bfa..b2acda3 100644 --- a/backend/config.py +++ b/backend/config.py @@ -125,6 +125,18 @@ class Settings(BaseSettings): # this many objects per frame to bound cost. SAM2_MAX_OBJECTS: int = 5 + # ── Real-time BOLO appearance matching ───────────────── + BOLO_REALTIME_ENABLED: bool = True + BOLO_MATCH_THRESHOLD: float = 0.82 + + # ── ALPR (license-plate OCR via EasyOCR) ─────────────── + ALPR_ENABLED: bool = True + ALPR_GPU: bool = False # CPU by default to avoid GPU contention with Ollama + ALPR_MIN_CONFIDENCE: float = 0.4 + + # ── Audio event detection (DSP; pluggable deep model) ── + AUDIO_DETECTION_ENABLED: bool = True + # ── Qdrant ──────────────────────────────────────────────── QDRANT_HOST: str = "localhost" QDRANT_PORT: int = 6333 diff --git a/backend/services/adaptive_thresholds.py b/backend/services/adaptive_thresholds.py new file mode 100644 index 0000000..243b999 --- /dev/null +++ b/backend/services/adaptive_thresholds.py @@ -0,0 +1,58 @@ +"""Adaptive thresholds cache β€” wires learned baselines into the hot loop. + +`baseline_learning_service.get_adaptive_thresholds` returns per-camera/zone +thresholds (mean + 2*std of learned-normal), but it is async and hits the DB β€” +too expensive to call per frame from the synchronous detection loop. This cache +bridges the gap: an async `refresh()` (called on a throttle from the monitoring +agent, which already holds a DB session) populates an in-memory table, and a +fast sync `get()` lets hot-loop code (e.g. temporal_behaviour) read a learned +threshold, transparently falling back to the caller's hard-coded default when no +fresh baseline exists. +""" +from __future__ import annotations + +import logging +import time +import uuid +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +_TTL_SECONDS = 900.0 # a cached threshold older than this is treated as absent + + +class AdaptiveThresholds: + def __init__(self) -> None: + # camera_id(str) -> {metric: value, "_t": fetched_at} + self._cache: Dict[str, Dict[str, Any]] = {} + + def get(self, camera_id: str, key: str, default: float) -> float: + """Return a fresh learned threshold for the camera, else `default`.""" + entry = self._cache.get(str(camera_id)) + if not entry: + return default + if time.time() - entry.get("_t", 0.0) > _TTL_SECONDS: + return default + val = entry.get(key) + return float(val) if isinstance(val, (int, float)) else default + + def source(self, camera_id: str) -> str: + entry = self._cache.get(str(camera_id)) + if not entry or time.time() - entry.get("_t", 0.0) > _TTL_SECONDS: + return "default" + return entry.get("source", "default") + + async def refresh(self, db, camera_id: str, zone_id: Optional[str] = None) -> None: + """Pull the latest learned thresholds for a camera into the cache.""" + try: + from backend.services.baseline_learning_service import baseline_learning_service + cam_uuid = uuid.UUID(str(camera_id)) + zone_uuid = uuid.UUID(str(zone_id)) if zone_id else None + thr = await baseline_learning_service.get_adaptive_thresholds(db, cam_uuid, zone_uuid) + thr["_t"] = time.time() + self._cache[str(camera_id)] = thr + except Exception as exc: # noqa: BLE001 β€” never let threshold refresh break the loop + logger.debug("adaptive_thresholds.refresh failed for %s: %s", camera_id, exc) + + +adaptive_thresholds = AdaptiveThresholds() diff --git a/backend/services/alpr_service.py b/backend/services/alpr_service.py new file mode 100644 index 0000000..b86120d --- /dev/null +++ b/backend/services/alpr_service.py @@ -0,0 +1,102 @@ +"""Real local ALPR (Automatic License-Plate Recognition) via EasyOCR. + +Replaces the slow, inconsistent vision-model plate path with a fast, local OCR +engine. Given a vehicle crop it reads candidate text, keeps the most plate-like +token (alphanumeric, plate-length), normalises it, and returns it for BOLO / +watchlist matching (`bolo_service.check_plate_match`) and VehicleSighting +logging. Lazy-loaded and gated by ALPR_ENABLED so it never affects startup; runs +on CPU by default to avoid GPU contention with Ollama. +""" +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, List, Optional, Sequence + +logger = logging.getLogger(__name__) + +_reader = None +_reader_lock = None +_PLATE_RE = re.compile(r"^[A-Z0-9]{4,9}$") +_ALNUM = re.compile(r"[^A-Z0-9]") + + +def _normalise(text: str) -> str: + return _ALNUM.sub("", (text or "").upper()) + + +class ALPRService: + def _get_reader(self): + global _reader, _reader_lock + from backend.config import settings + if not getattr(settings, "ALPR_ENABLED", True): + return None + import threading + if _reader_lock is None: + _reader_lock = threading.Lock() + with _reader_lock: + if _reader is None: + try: + import easyocr + use_gpu = bool(getattr(settings, "ALPR_GPU", False)) + _reader = easyocr.Reader(["en"], gpu=use_gpu, verbose=False) + logger.info("ALPR (EasyOCR) reader loaded (gpu=%s)", use_gpu) + except Exception as exc: # noqa: BLE001 + logger.warning("ALPR reader load failed: %s", exc) + _reader = None + return _reader + + def available(self) -> bool: + return self._get_reader() is not None + + def read_plate(self, frame_bgr, bbox: Optional[Sequence[float]] = None) -> Optional[Dict[str, Any]]: + """Read the most plate-like text from a (vehicle) crop. + + Returns {plate, confidence, raw} for the best candidate, or None. + """ + reader = self._get_reader() + if reader is None or frame_bgr is None: + return None + from backend.config import settings + min_conf = float(getattr(settings, "ALPR_MIN_CONFIDENCE", 0.4)) + + crop = frame_bgr + try: + if bbox is not None: + x1, y1, x2, y2 = [int(v) for v in bbox] + crop = frame_bgr[max(0, y1):max(0, y2), max(0, x1):max(0, x2)] + if crop is None or crop.size == 0: + return None + detections = reader.readtext(crop) # [(box, text, conf), ...] + except Exception as exc: # noqa: BLE001 + logger.debug("ALPR readtext failed: %s", exc) + return None + + best: Optional[Dict[str, Any]] = None + for det in detections or []: + try: + _box, text, conf = det + except Exception: + continue + norm = _normalise(text) + if not _PLATE_RE.match(norm) or conf < min_conf: + continue + if best is None or conf > best["confidence"]: + best = {"plate": norm, "confidence": round(float(conf), 3), "raw": text} + return best + + async def read_and_match(self, frame_bgr, bbox=None) -> Dict[str, Any]: + """Read a plate and check it against active vehicle BOLOs.""" + result = self.read_plate(frame_bgr, bbox) + if not result: + return {"plate": None, "matches": []} + try: + from backend.services.bolo_service import bolo_service + matches = await bolo_service.check_plate_match(result["plate"]) + except Exception as exc: # noqa: BLE001 + logger.debug("ALPR BOLO match failed: %s", exc) + matches = [] + return {**result, "matches": matches} + + +alpr_service = ALPRService() diff --git a/backend/services/audio_detection_service.py b/backend/services/audio_detection_service.py new file mode 100644 index 0000000..2b90f98 --- /dev/null +++ b/backend/services/audio_detection_service.py @@ -0,0 +1,129 @@ +"""Real audio event detection (DSP) β€” replaces the video-inference stub. + +The audio sentinel previously *inferred* sound from the video scene (a placeholder +β€” there was no audio at all). This analyses ACTUAL audio: it extracts acoustic +features with librosa and classifies high-value security sounds by their physical +signature: + + - gunshot : a loud, broadband, impulsive transient (sharp onset, high flatness) + - glass_break : a high-frequency burst (very high spectral centroid + ZCR) + - scream : sustained, loud, high-pitched harmonic energy (low flatness) + - alarm/siren : a sustained narrow-band tone (very low flatness, stable pitch) + +Deterministic and dependency-light (librosa only). `classify()` is the pluggable +seam β€” a deep model (PANNs/YAMNet) can replace the heuristic later without +changing callers. Audio ingestion (ffmpeg from an RTSP audio track or a mic) is +provided but needs a real source; the analysis path is unit-tested on buffers. +""" +from __future__ import annotations + +import io +import logging +from typing import Any, Dict, List, Optional + +import numpy as np + +logger = logging.getLogger(__name__) + + +def _features(y: np.ndarray, sr: int) -> Dict[str, float]: + import librosa + if y.size == 0: + return {} + y = y.astype(np.float32) + hop = 512 + rms = librosa.feature.rms(y=y, hop_length=hop)[0] + onset = librosa.onset.onset_strength(y=y, sr=sr) + + # Spectral character is computed on the LOUDEST window, not averaged over the + # whole (often mostly-silent) buffer β€” otherwise a short burst is diluted. + seg = y + if rms.size: + peak_frame = int(np.argmax(rms)) + c = peak_frame * hop + half = max(int(0.15 * sr), 256) + seg = y[max(0, c - half): c + half] + if seg.size < 256: + seg = y + + cent = librosa.feature.spectral_centroid(y=seg, sr=sr)[0] + flat = librosa.feature.spectral_flatness(y=seg)[0] + zcr = librosa.feature.zero_crossing_rate(y=seg)[0] + return { + "peak_rms": float(np.max(rms)) if rms.size else 0.0, + "mean_rms": float(np.mean(rms)) if rms.size else 0.0, + "centroid_hz": float(np.mean(cent)), + "flatness": float(np.mean(flat)), + "zcr": float(np.mean(zcr)), + "onset_peak": float(np.max(onset)) if onset.size else 0.0, + "duration_s": float(len(y) / sr), + } + + +class AudioDetectionService: + # Loudness gate β€” ignore quiet ambient audio entirely. + _RMS_GATE = 0.05 + + def classify(self, y: np.ndarray, sr: int) -> List[Dict[str, Any]]: + """Classify a mono audio buffer into security-relevant sound events.""" + f = _features(y, sr) + if not f or f["peak_rms"] < self._RMS_GATE: + return [] + + events: List[Dict[str, Any]] = [] + peak, cent, flat, zcr, onset = ( + f["peak_rms"], f["centroid_hz"], f["flatness"], f["zcr"], f["onset_peak"] + ) + + # Very high-frequency burst β†’ breaking glass (checked first; its high + # centroid is the distinguishing signature vs a broadband gunshot). + if cent >= 5000 and zcr >= 0.15: + events.append({"label": "glass_break", "severity": "high", + "confidence": round(min(0.9, 0.4 + cent / 12000 + zcr), 2)}) + # Loud, broadband, impulsive β†’ gunshot. + elif peak >= 0.25 and flat >= 0.20 and onset >= 1.0: + events.append({"label": "gunshot", "severity": "critical", + "confidence": round(min(0.95, 0.5 + flat + peak / 2), 2)}) + # Sustained, loud, high-pitched, harmonic β†’ scream/shout. + elif peak >= 0.15 and 1200 <= cent <= 4500 and flat < 0.12 and f["duration_s"] >= 0.4: + events.append({"label": "scream", "severity": "high", + "confidence": round(min(0.9, 0.4 + peak), 2)}) + # Sustained narrow-band tone β†’ alarm / siren. + elif flat < 0.02 and f["mean_rms"] >= 0.1: + events.append({"label": "alarm", "severity": "medium", + "confidence": round(min(0.9, 0.5 + (0.02 - flat) * 10), 2)}) + + for e in events: + e["features"] = {k: round(v, 4) for k, v in f.items()} + return events + + def analyze_bytes(self, wav_bytes: bytes) -> List[Dict[str, Any]]: + try: + import soundfile as sf + y, sr = sf.read(io.BytesIO(wav_bytes), dtype="float32") + if y.ndim > 1: + y = y.mean(axis=1) # mono + return self.classify(y, sr) + except Exception as exc: # noqa: BLE001 + logger.warning("audio analyze_bytes failed: %s", exc) + return [] + + def extract_audio_from_rtsp(self, url: str, seconds: float = 3.0, sr: int = 16000) -> Optional[np.ndarray]: + """Pull a few seconds of audio from an RTSP stream via ffmpeg (needs an + audio track on the stream). Returns a mono float32 buffer, or None.""" + import subprocess + try: + proc = subprocess.run( + ["ffmpeg", "-i", url, "-t", str(seconds), "-ac", "1", "-ar", str(sr), + "-f", "f32le", "-loglevel", "quiet", "pipe:1"], + capture_output=True, timeout=seconds + 10, + ) + if proc.returncode != 0 or not proc.stdout: + return None + return np.frombuffer(proc.stdout, dtype=np.float32) + except Exception as exc: # noqa: BLE001 + logger.debug("rtsp audio extract failed: %s", exc) + return None + + +audio_detection_service = AudioDetectionService() diff --git a/backend/services/bolo_matcher.py b/backend/services/bolo_matcher.py new file mode 100644 index 0000000..18c6603 --- /dev/null +++ b/backend/services/bolo_matcher.py @@ -0,0 +1,108 @@ +"""Real-time BOLO appearance matching. + +`bolo_service` already does vehicle-plate matching (Levenshtein). This adds the +missing half: matching detected PEOPLE against active person BOLOs by CLIP +appearance embedding. Active person BOLOs (with an enrolled appearance embedding +stored in their JSONB ``description``) are cached in memory and refreshed on a +throttle; every new person track is embedded once (reusing +`appearance_embedder.appearance_embedding`) and matched by cosine similarity. + +A hit becomes a high-severity threat that flows through the normal (verified) +alert path β€” turning the BOLO list from a static database into an active watch. +""" +from __future__ import annotations + +import logging +import time +from typing import Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + +_TTL_SECONDS = 300.0 + + +class BoloMatcher: + def __init__(self) -> None: + self._cache: List[Dict[str, Any]] = [] # {bolo_id, embedding, severity, reason} + self._cache_t: float = 0.0 + self._checked: Dict[str, set] = {} # camera_id -> track_ids already scanned + + def has_active(self) -> bool: + return bool(self._cache) and (time.time() - self._cache_t) <= _TTL_SECONDS + + async def refresh(self, db) -> None: + """Load active person BOLOs (with an enrolled appearance embedding).""" + try: + from backend.services.bolo_service import bolo_service + bolos = await bolo_service.get_active_bolos(bolo_type="person") + except Exception as exc: # noqa: BLE001 + logger.debug("bolo_matcher.refresh failed: %s", exc) + return + cache: List[Dict[str, Any]] = [] + for b in bolos or []: + desc = b.get("description") or {} + emb = desc.get("appearance_embedding") if isinstance(desc, dict) else None + if emb: + cache.append({ + "bolo_id": b.get("id"), + "embedding": emb, + "severity": b.get("severity", "high"), + "reason": b.get("reason") or "BOLO appearance match", + }) + self._cache = cache + self._cache_t = time.time() + + def match(self, embedding: List[float], threshold: float = 0.82) -> List[Dict[str, Any]]: + from backend.services.appearance_embedder import cosine_similarity + out: List[Dict[str, Any]] = [] + for entry in self._cache: + sim = cosine_similarity(embedding, entry["embedding"]) + if sim >= threshold: + out.append({**entry, "similarity": sim}) + out.sort(key=lambda m: m["similarity"], reverse=True) + return out + + def scan_frame(self, frame_bgr, detections: Dict[str, Any], camera_id: str, + threshold: Optional[float] = None) -> List[Dict[str, Any]]: + """Embed new person tracks and return BOLO-match threats.""" + if not self.has_active(): + return [] + try: + from backend.config import settings + thr = threshold if threshold is not None else float(getattr(settings, "BOLO_MATCH_THRESHOLD", 0.82)) + except Exception: + thr = threshold if threshold is not None else 0.82 + from backend.services.appearance_embedder import appearance_embedding + + checked = self._checked.setdefault(str(camera_id), set()) + present = {d.get("track_id") for d in detections.get("detections", [])} + # prune track_ids no longer present so a returning subject re-scans + checked &= present + + out: List[Dict[str, Any]] = [] + for d in detections.get("detections", []) if isinstance(detections, dict) else []: + if (d.get("class") or "").lower() != "person": + continue + tid = d.get("track_id") + if tid is None or tid in checked or not d.get("bbox"): + continue + checked.add(tid) + emb = appearance_embedding(frame_bgr, d["bbox"]) + if not emb: + continue + matches = self.match(emb, thr) + if matches: + best = matches[0] + out.append({ + "signature": "bolo_person_match", + "description": f"appearance matches active BOLO ({best['reason']})", + "severity": best.get("severity", "high"), + "confidence": round(float(best["similarity"]), 2), + "detection_method": "bolo", + "track_id": tid, + "bolo_id": str(best.get("bolo_id")), + }) + return out + + +bolo_matcher = BoloMatcher() diff --git a/backend/services/bolo_service.py b/backend/services/bolo_service.py index 08a76ae..d93fbd2 100644 --- a/backend/services/bolo_service.py +++ b/backend/services/bolo_service.py @@ -192,6 +192,31 @@ async def deactivate_bolo(self, bolo_id: uuid.UUID) -> bool: logger.error("Failed to deactivate BOLO %s: %s", bolo_id, exc, exc_info=True) return False + async def enroll_appearance(self, bolo_id: uuid.UUID, embedding: List[float]) -> bool: + """Store a CLIP appearance embedding on a person BOLO (in description JSONB) + so the real-time matcher can match detected people against it.""" + try: + from backend.database import async_session + from backend.models.phase2_models import BOLOEntry + + async with async_session() as session: + entry = await session.get(BOLOEntry, bolo_id) + if entry is None: + return False + desc = dict(entry.description or {}) + desc["appearance_embedding"] = list(embedding) + entry.description = desc + # JSONB needs an explicit reassignment to be flagged dirty + from sqlalchemy.orm.attributes import flag_modified + flag_modified(entry, "description") + entry.updated_at = datetime.now(timezone.utc) + await session.commit() + logger.info("BOLO %s enrolled with %d-d appearance embedding", bolo_id, len(embedding)) + return True + except Exception as exc: + logger.error("Failed to enroll BOLO appearance %s: %s", bolo_id, exc, exc_info=True) + return False + # ── Fuzzy plate matching ───────────────────────────────────── async def check_plate_match(self, plate_text: str) -> List[Dict[str, Any]]: diff --git a/backend/services/compliance_dashboard_service.py b/backend/services/compliance_dashboard_service.py index 4fe283b..0710c9e 100644 --- a/backend/services/compliance_dashboard_service.py +++ b/backend/services/compliance_dashboard_service.py @@ -715,6 +715,53 @@ async def get_compliance_history( for r in rows ] + async def forecast_compliance( + self, db: AsyncSession, framework: str = "gdpr", target: float = 0.8, limit: int = 12, + ) -> Dict[str, Any]: + """Project the compliance score forward from its recent trend and estimate + when (if ever) it will fall below `target` β€” proactive vs. reactive.""" + history = await self.get_compliance_history(db, framework=framework, limit=limit) + pts = [] + for h in history: + if h.get("assessed_at") and h.get("overall_score") is not None: + try: + pts.append((datetime.fromisoformat(h["assessed_at"]), float(h["overall_score"]))) + except Exception: + continue + pts.sort(key=lambda p: p[0]) + if len(pts) < 3: + return {"framework": framework, "status": "insufficient_history", "sample_count": len(pts)} + + t0 = pts[0][0] + xs = [(p[0] - t0).total_seconds() / 86400.0 for p in pts] # days + ys = [p[1] for p in pts] + n = len(xs) + mx, my = sum(xs) / n, sum(ys) / n + denom = sum((x - mx) ** 2 for x in xs) or 1e-9 + slope = sum((xs[i] - mx) * (ys[i] - my) for i in range(n)) / denom # score/day + current = ys[-1] + + breach_in_days = None + if slope < -1e-6 and current > target: + breach_in_days = round((current - target) / (-slope), 1) + projected_30d = round(max(0.0, min(1.0, current + slope * 30)), 3) + + if current < target: + status = "below_target" + elif breach_in_days is not None and breach_in_days <= 30: + status = "at_risk" + else: + status = "on_track" + return { + "framework": framework, "target": target, + "current_score": round(current, 3), + "trend_per_day": round(slope, 5), + "projected_breach_in_days": breach_in_days, + "projected_30d_score": projected_30d, + "status": status, + "sample_count": n, + } + async def get_issues( self, db: AsyncSession, severity: Optional[str] = None, ) -> List[Dict[str, Any]]: diff --git a/backend/services/escalation_tracker.py b/backend/services/escalation_tracker.py new file mode 100644 index 0000000..29a0747 --- /dev/null +++ b/backend/services/escalation_tracker.py @@ -0,0 +1,85 @@ +"""Per-entity threat-escalation chains. + +Individual behaviours (loitering, running, a fall, a BOLO hit) are scored in +isolation by the detection layer. But a *sequence* on the same entity β€” +loiter β†’ test-a-door β†’ approach β€” is far more telling than any single step. +This tracker ingests the track-bearing threats already produced each frame, +maintains a short per-(camera, track) history, and emits a CRITICAL +``escalation_chain`` threat when an entity shows several DISTINCT, escalating +behaviours inside a time window β€” turning a string of low/medium signals into one +high-confidence proactive warning. +""" +from __future__ import annotations + +import logging +from collections import deque +from typing import Any, Deque, Dict, List, Tuple + +logger = logging.getLogger(__name__) + +# Behaviour β†’ escalation rank (higher = closer to a hostile act). +_RANK: Dict[str, int] = { + "loitering": 1, "tailgating": 2, "running": 2, "abandoned_object": 2, + "tamper": 2, "evasive": 2, "intrusion": 3, "fall": 3, "bolo_person_match": 3, + "concealed_carry": 4, "pre_assault": 4, "weapon": 4, "aggression": 4, +} + +_WINDOW_S = 300.0 # escalation steps must fall within this window +_MIN_STEPS = 3 # distinct behaviours to count as a chain +_COOLDOWN_S = 120.0 # don't re-fire a chain for the same track this often + + +class EscalationTracker: + def __init__(self) -> None: + self._tracks: Dict[Tuple[str, Any], Deque[Tuple[float, str, int]]] = {} + self._cooldowns: Dict[Tuple[str, Any], float] = {} + + def observe(self, camera_id: str, threats: List[Dict[str, Any]], t: float) -> List[Dict[str, Any]]: + """Record this frame's threats and return any escalation_chain threats.""" + out: List[Dict[str, Any]] = [] + for thr in threats or []: + tid = thr.get("track_id") + sig = thr.get("signature") + if tid is None or not sig or sig == "escalation_chain": + continue + rank = _RANK.get(sig) + if rank is None: + continue + key = (str(camera_id), tid) + dq = self._tracks.setdefault(key, deque()) + dq.append((t, sig, rank)) + while dq and t - dq[0][0] > _WINDOW_S: + dq.popleft() + + # distinct behaviours seen in the window (keep highest rank per sig) + distinct: Dict[str, int] = {} + for (_tt, s, r) in dq: + distinct[s] = max(distinct.get(s, 0), r) + if len(distinct) < _MIN_STEPS: + continue + ranks = sorted(distinct.values()) + if max(ranks) <= min(ranks): + continue # no actual escalation, just repetition + if t - self._cooldowns.get(key, -1e9) < _COOLDOWN_S: + continue + self._cooldowns[key] = t + chain = " β†’ ".join(s for s, _ in sorted(distinct.items(), key=lambda kv: kv[1])) + out.append({ + "signature": "escalation_chain", + "description": f"behavioural escalation on track {tid}: {chain}", + "severity": "critical", + "confidence": round(min(0.95, 0.7 + 0.05 * len(distinct)), 2), + "detection_method": "escalation", + "track_id": tid, + }) + logger.info("escalation_chain on camera %s track %s: %s", camera_id, tid, chain) + return out + + def prune(self, camera_id: str, active_track_ids: set) -> None: + """Drop history for tracks no longer present (called opportunistically).""" + for key in [k for k in self._tracks if k[0] == str(camera_id) and k[1] not in active_track_ids]: + self._tracks.pop(key, None) + self._cooldowns.pop(key, None) + + +escalation_tracker = EscalationTracker() diff --git a/backend/services/forensic_search_service.py b/backend/services/forensic_search_service.py index 771f1ae..1286bd4 100644 --- a/backend/services/forensic_search_service.py +++ b/backend/services/forensic_search_service.py @@ -120,6 +120,55 @@ async def search_by_similarity(self, query_text: str, top_k: int = 20, logger.error("Similarity search failed: %s", e) return {"query": query_text, "result_count": 0, "results": [], "error": str(e)} + async def search_objects_by_image( + self, frame_bgr, bbox=None, camera_id: str = None, + time_from=None, time_to=None, top_k: int = 20, + ) -> dict: + """Image-based "looks-like" search: given a query image (optionally a + bbox crop), find the most visually similar people/objects across cameras. + + CLIP image and text embeddings share a space, so this reuses the same + ``object_crops`` collection that `search_objects_by_text` queries β€” just + embedding the pixels instead of a phrase. Degrades to an empty result set + when CLIP/Qdrant is unavailable. + """ + try: + from backend.services.clip_embedder import clip_embedder + from backend.services.vector_store import vector_store + except Exception: + return {"query": "image", "result_count": 0, "results": []} + + crop = frame_bgr + try: + if bbox is not None: + x1, y1, x2, y2 = [int(v) for v in bbox] + crop = frame_bgr[max(0, y1):max(0, y2), max(0, x1):max(0, x2)] + vec = clip_embedder.embed_frame_sync(crop) if crop is not None and crop.size else [] + vec = list(vec) if vec else [] + except Exception: + vec = [] + if not vec: + return {"query": "image", "result_count": 0, "results": []} + + filters = {"camera_id": str(camera_id)} if camera_id else None + hits = await vector_store.search_by_vector( + vector=vec, top_k=max(int(top_k), 1), filters=filters, collection="object_crops", + ) + tf, tt = _to_epoch(time_from), _to_epoch(time_to) + results = [] + for h in hits or []: + ts = h.get("timestamp") + if tf is not None and ts is not None and ts < tf: + continue + if tt is not None and ts is not None and ts > tt: + continue + results.append({ + "entity_id": h.get("entity_id"), "camera_id": h.get("camera_id"), + "zone_id": h.get("zone_id"), "timestamp": ts, + "behavior": h.get("behavior"), "score": h.get("score"), + }) + return {"query": "image", "result_count": len(results), "results": results} + async def search_objects_by_text( self, query_text: str, camera_id: str = None, time_from=None, time_to=None, top_k: int = 20, diff --git a/backend/services/temporal_behavior.py b/backend/services/temporal_behavior.py index 4c24f79..68f3de6 100644 --- a/backend/services/temporal_behavior.py +++ b/backend/services/temporal_behavior.py @@ -181,11 +181,16 @@ def _person_behaviours(self, cam_id, tid, st, t, w_frame, h_frame, diag) -> List f"person track {tid}: sustained speed {speed_frac:.2f} frame-widths/s", tid)) # --- Loitering: long dwell in a small area --- + # The dwell threshold is learned per camera/zone/time-slot when a baseline + # exists (cuts false positives in naturally-busy areas); falls back to the + # static _LOITER_S otherwise. + from backend.services.adaptive_thresholds import adaptive_thresholds + loiter_s = adaptive_thresholds.get(cam_id, "dwell_time_threshold", _LOITER_S) dwell = st.last_seen - st.first_seen - if dwell >= _LOITER_S: + if dwell >= loiter_s: spread = self._spatial_spread(hist) / diag if spread <= _LOITER_RADIUS_FRAC and self._fire(st, "loitering", t): - conf = min(0.95, 0.5 + (dwell - _LOITER_S) / max(_LOITER_S, 1) * 0.4) + conf = min(0.95, 0.5 + (dwell - loiter_s) / max(loiter_s, 1) * 0.4) out.append(self._mk("loitering", "low", round(conf, 2), f"person track {tid}: dwelling {int(dwell)}s within a small area (spread {spread:.2f})", tid)) return out diff --git a/backend/services/trajectory_predictor.py b/backend/services/trajectory_predictor.py new file mode 100644 index 0000000..8c4b25e --- /dev/null +++ b/backend/services/trajectory_predictor.py @@ -0,0 +1,77 @@ +"""Trajectory prediction β€” turn past motion into a near-future forecast. + +The tracker records where entities have BEEN; this predicts where they are +GOING. A constant-velocity estimate (least-squares over recent positions, robust +to per-frame jitter) yields the predicted position at a horizon, current speed +and heading, and the ETA at which the path passes closest to a point of interest +(e.g. a secure door) β€” enabling proactive alerts ("if this person continues they +reach the server-room door in ~8s") instead of purely reactive ones. +""" +from __future__ import annotations + +import math +from typing import Any, Dict, List, Optional, Sequence, Tuple + + +def _velocity(points: Sequence[Tuple[float, float, float]]) -> Tuple[float, float, float, float]: + """Least-squares constant velocity over (t, x, y). Returns (x0,y0,vx,vy) at the latest t.""" + n = len(points) + t0 = points[-1][0] + ts = [p[0] - t0 for p in points] # center on latest sample + mean_t = sum(ts) / n + denom = sum((t - mean_t) ** 2 for t in ts) or 1e-6 + + def slope_intercept(vals): + mean_v = sum(vals) / n + slope = sum((ts[i] - mean_t) * (vals[i] - mean_v) for i in range(n)) / denom + intercept = mean_v - slope * mean_t # value at t==0 (i.e. latest time) + return slope, intercept + + vx, x0 = slope_intercept([p[1] for p in points]) + vy, y0 = slope_intercept([p[2] for p in points]) + return x0, y0, vx, vy + + +def predict(history: Sequence[Tuple[float, float, float]], horizon_s: float = 5.0) -> Optional[Dict[str, Any]]: + """Predict motion from (t, x, y) history. + + Returns {position:[x,y], velocity:[vx,vy], speed, heading_deg} for `horizon_s` + into the future, or None if there isn't enough history. + """ + pts = [(float(t), float(x), float(y)) for (t, x, y) in history if t is not None] + if len(pts) < 3: + return None + pts = pts[-12:] # recent window only + x0, y0, vx, vy = _velocity(pts) + px, py = x0 + vx * horizon_s, y0 + vy * horizon_s + speed = math.hypot(vx, vy) + heading = (math.degrees(math.atan2(vy, vx)) + 360.0) % 360.0 + return { + "position": [round(px, 1), round(py, 1)], + "velocity": [round(vx, 3), round(vy, 3)], + "speed": round(speed, 3), + "heading_deg": round(heading, 1), + "current": [round(x0, 1), round(y0, 1)], + } + + +def eta_to_point(history: Sequence[Tuple[float, float, float]], target: Tuple[float, float], + max_horizon_s: float = 30.0, hit_radius: float = 40.0) -> Optional[float]: + """Seconds until the predicted path comes within `hit_radius` of `target`, + or None if it never does within `max_horizon_s` (or motion is too slow).""" + pred = predict(history, horizon_s=0.0) + if pred is None: + return None + x0, y0 = pred["current"] + vx, vy = pred["velocity"] + if math.hypot(vx, vy) < 1e-3: + return None + # sample the predicted ray; cheap and robust + step = 0.25 + t = 0.0 + while t <= max_horizon_s: + x, y = x0 + vx * t, y0 + vy * t + if math.hypot(x - target[0], y - target[1]) <= hit_radius: + return round(t, 2) + t += step + return None diff --git a/frontend/src/app/admin/page.tsx b/frontend/src/app/admin/page.tsx index 00d3f96..64cb143 100644 --- a/frontend/src/app/admin/page.tsx +++ b/frontend/src/app/admin/page.tsx @@ -69,7 +69,7 @@ const PLAN_LABELS: Record = { }; const PLAN_COLORS: Record = { - basic: "border-gray-700 text-gray-400 bg-gray-800/40", + basic: "border-border-strong text-muted-foreground bg-surface-3/40", professional: "border-blue-700/60 text-blue-400 bg-blue-900/20", enterprise: "border-cyan-700/60 text-cyan-400 bg-cyan-900/20", }; @@ -126,13 +126,13 @@ function StatCard({ accent: string; }) { return ( -
+
-

{label}

-

{value}

+

{label}

+

{value}

); @@ -154,12 +154,12 @@ function UsageBar({ return (
- {label} - + {label} + {used} / {unlimited ? "∞" : max}
-
+
{!unlimited && (
-
+
{/* Header */} -
-

+
+

Create Organisation

@@ -263,7 +263,7 @@ function CreateOrgModal({ onClose, onCreated }: CreateOrgModalProps) {
{/* Name */}
-
{/* Slug */}
-
{/* Plan */}
-
{/* Limits row */}
-
-
@@ -357,7 +357,7 @@ function CreateOrgModal({ onClose, onCreated }: CreateOrgModalProps) { @@ -390,9 +390,9 @@ function OrganisationsTab({ if (tenants.length === 0) { return (
- -

No tenants configured

-

+ +

No tenants configured

+

Create an organisation above to get started.

@@ -405,15 +405,15 @@ function OrganisationsTab({
{/* Header */}
-

{tenant.name}

-

{tenant.slug}

+

{tenant.name}

+

{tenant.slug}

{/* Stats */} -
+
-

Users

-

+

Users

+

{tenant.user_count} - + /{tenant.max_users === -1 ? "∞" : tenant.max_users}

-
-

Sites

-

+

+

Sites

+

{tenant.site_count} - + /{tenant.max_sites === -1 ? "∞" : tenant.max_sites}

-

Cameras

-

{tenant.camera_count}

+

Cameras

+

{tenant.camera_count}

{/* Created */} -

+

Created {formatDate(tenant.created_at)}

{/* Actions */}
- @@ -494,36 +494,36 @@ function OrganisationsTab({ function UsageTab({ tenants }: { tenants: Tenant[] }) { return ( -
+
- - + - - - - - - + {tenants.map((tenant) => ( - + ))}
+
Organisation + Users + Sites + Cameras + API Calls (24h) + Storage
-

{tenant.name}

-

{tenant.slug}

+

{tenant.name}

+

{tenant.slug}

 - {tenant.camera_count} + {tenant.camera_count} - 0 + 0 - β€” + β€”
{tenants.length === 0 && ( -
No organisations found.
+
No organisations found.
)}
); @@ -626,14 +626,14 @@ function BrandingTab({ tenants }: { tenants: Tenant[] }) {
{/* Org selector */}
-
{/* Logo URL */}
-