fix: Enhance blob management by adding clear_all_blobs method and updating file_id handling

azure.yaml

@@ -55,13 +55,13 @@ hooks:
         python -m pip install --quiet --upgrade pip
         pip install --quiet -r requirements.txt
 
-        # # ── Step 1: Create AI agent ──
-        # Write-Host ""
-        # Write-Host "Setting up AI agent..." -ForegroundColor Yellow
-        # & ./infra/scripts/setup-agent.ps1
-        # if ($LASTEXITCODE -ne 0) {
-        #   Write-Host "WARNING: Agent setup failed — you can retry later with: ./infra/scripts/setup-agent.ps1" -ForegroundColor Yellow
-        # }
+        # ── Step 1: Create AI agent ──
+        Write-Host ""
+        Write-Host "Setting up AI agent..." -ForegroundColor Yellow
+        & ./infra/scripts/setup-agent.ps1
+        if ($LASTEXITCODE -ne 0) {
+          Write-Host "WARNING: Agent setup failed — you can retry later with: ./infra/scripts/setup-agent.ps1" -ForegroundColor Yellow
+        }
 
         # ── Step 1.5: Grant API identity access to SQL ──
         Write-Host ""
@@ -103,9 +103,9 @@ hooks:
         python -m pip install --quiet --upgrade pip
         pip install --quiet -r requirements.txt
 
-        # echo ""
-        # echo "Setting up AI agent..."
-        # pwsh -File ./infra/scripts/setup-agent.ps1 || echo "WARNING: Agent setup failed — retry with: ./infra/scripts/setup-agent.ps1"
+        echo ""
+        echo "Setting up AI agent..."
+        pwsh -File ./infra/scripts/setup-agent.ps1 || echo "WARNING: Agent setup failed — retry with: ./infra/scripts/setup-agent.ps1"
 
         echo ""
         echo "Granting API identity SQL access..."

src/api/modules/ingestion/azure_storage.py

@@ -345,10 +345,11 @@ def delete_file_data(self, file_id: str, filename: str, doc_ids: list[str]) -> d
                 except Exception:
                     pass
 
-                # Delete doc-level blobs (documents/{doc_id}.json)
+                # Delete doc-level blobs ({doc_id}.json at container root,
+                # matching the path used by upload_to_blob)
                 for doc_id in doc_ids:
                     try:
-                        blob = container.get_blob_client(f"documents/{doc_id}.json")
+                        blob = container.get_blob_client(f"{doc_id}.json")
                         blob.delete_blob()
                     except Exception:
                         pass
@@ -387,5 +388,32 @@ def delete_file_data(self, file_id: str, filename: str, doc_ids: list[str]) -> d
         logger.info(f"Cleanup for file '{file_id}': {result}")
         return result
 
+    def clear_all_blobs(self) -> int:
+        """Delete every blob in the storage container (raw/, extracted/, documents/, etc.).
+
+        Used when clearing all data before loading a new scenario so that stale
+        files from a previous scenario are not left behind in storage.
+        Returns the number of blobs deleted.
+        """
+        settings = get_settings()
+        if not settings.azure_storage_account:
+            return 0
+
+        deleted = 0
+        try:
+            blob_service = self._get_blob_client()
+            container = blob_service.get_container_client(settings.azure_storage_container)
+            blob_names = [b.name for b in container.list_blobs()]
+            for name in blob_names:
+                try:
+                    container.delete_blob(name)
+                    deleted += 1
+                except Exception:
+                    pass
+            logger.info(f"Cleared all blobs from container '{settings.azure_storage_container}': deleted {deleted}")
+        except Exception as e:
+            logger.warning(f"Failed to clear blobs: {e}")
+        return deleted
+
 
 azure_storage_service = AzureStorageService()

src/api/modules/ingestion/service.py

@@ -374,7 +374,7 @@ def _track_file(self, filename: str, data: list[dict]):
         """
         from datetime import datetime
 
-        file_id = filename.rsplit(".", 1)[0]
+        file_id = filename.rsplit(".", 1)[0].replace(" ", "_")
 
         summary = ""
         keywords: list[str] = []
@@ -739,7 +739,7 @@ def load_json_file(self, file_path: str) -> IngestionResult:
         # Track doc_ids for reliable delete
         ingested_ids = [item["id"] for item in raw_data]
         from datetime import datetime
-        file_id = filename.rsplit(".", 1)[0]
+        file_id = filename.rsplit(".", 1)[0].replace(" ", "_")
         self._uploaded_files[file_id] = UploadedFile(
             id=file_id,
             filename=filename,
@@ -778,7 +778,7 @@ def load_json_data(self, data: list[dict], filename: str = "uploaded_data.json")
 
         # Track file immediately (in-memory) so it appears in the file list right away
         from datetime import datetime
-        file_id = filename.rsplit(".", 1)[0]
+        file_id = filename.rsplit(".", 1)[0].replace(" ", "_")
         ingested_ids = [item["id"] for item in data]
 
         # Preserve existing status/error if file already exists (e.g., set to "processing" by upload)
@@ -976,6 +976,14 @@ def clear(self):
         except Exception as e:
             logger.warning(f"Failed to purge queues: {e}")
 
+        # Delete all blobs (raw/, extracted/, documents/) so stale files from a
+        # previous scenario are not left behind in storage.
+        try:
+            from src.api.modules.ingestion.azure_storage import azure_storage_service
+            azure_storage_service.clear_all_blobs()
+        except Exception as e:
+            logger.warning(f"Failed to clear blobs: {e}")
+
     def delete_file(self, file_id: str) -> bool:
         """Delete an uploaded file and all its documents."""
         self._ensure_loaded()

src/api/storage/sql_service.py

@@ -106,6 +106,7 @@ def _create_tables(self):
                 summary NVARCHAR(MAX),
                 keywords NVARCHAR(MAX),
                 filter_values NVARCHAR(MAX),
+                doc_ids NVARCHAR(MAX),
                 uploaded_at NVARCHAR(50)
             )
         """)
@@ -335,21 +336,23 @@ def save_uploaded_file(self, file_data: dict) -> bool:
                 MERGE uploaded_files AS target
                 USING (SELECT ? AS id) AS source ON target.id = source.id
                 WHEN MATCHED THEN UPDATE SET
-                    filename=?, doc_count=?, summary=?, keywords=?, filter_values=?, uploaded_at=?
+                    filename=?, doc_count=?, summary=?, keywords=?, filter_values=?, doc_ids=?, uploaded_at=?
                 WHEN NOT MATCHED THEN INSERT
-                    (id, filename, doc_count, summary, keywords, filter_values, uploaded_at)
-                    VALUES (?, ?, ?, ?, ?, ?, ?);
+                    (id, filename, doc_count, summary, keywords, filter_values, doc_ids, uploaded_at)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?);
                 """,
                 file_data["id"],
                 file_data.get("filename", ""), file_data.get("doc_count", 0),
                 file_data.get("summary", ""), json.dumps(file_data.get("keywords", [])),
                 json.dumps(file_data.get("filter_values", {})),
+                json.dumps(file_data.get("doc_ids", [])),
                 file_data.get("uploaded_at", ""),
                 # INSERT
                 file_data["id"],
                 file_data.get("filename", ""), file_data.get("doc_count", 0),
                 file_data.get("summary", ""), json.dumps(file_data.get("keywords", [])),
                 json.dumps(file_data.get("filter_values", {})),
+                json.dumps(file_data.get("doc_ids", [])),
                 file_data.get("uploaded_at", ""),
             )
             conn.commit()
@@ -366,7 +369,7 @@ def load_all_uploaded_files(self) -> list[dict]:
         try:
             conn = self._get_connection()
             cursor = conn.cursor()
-            cursor.execute("SELECT id, filename, doc_count, summary, keywords, filter_values, uploaded_at FROM uploaded_files")
+            cursor.execute("SELECT id, filename, doc_count, summary, keywords, filter_values, doc_ids, uploaded_at FROM uploaded_files")
             rows = cursor.fetchall()
             conn.close()
             return [
@@ -375,7 +378,8 @@ def load_all_uploaded_files(self) -> list[dict]:
                     "summary": r[3],
                     "keywords": json.loads(r[4]) if r[4] else [],
                     "filter_values": json.loads(r[5]) if r[5] else {},
-                    "uploaded_at": r[6] or "",
+                    "doc_ids": json.loads(r[6]) if r[6] else [],
+                    "uploaded_at": r[7] or "",
                 }
                 for r in rows
             ]