diff --git a/azure.yaml b/azure.yaml index c1dd7883b..fb746812d 100644 --- a/azure.yaml +++ b/azure.yaml @@ -55,13 +55,13 @@ hooks: python -m pip install --quiet --upgrade pip pip install --quiet -r requirements.txt - # # ── Step 1: Create AI agent ── - # Write-Host "" - # Write-Host "Setting up AI agent..." -ForegroundColor Yellow - # & ./infra/scripts/setup-agent.ps1 - # if ($LASTEXITCODE -ne 0) { - # Write-Host "WARNING: Agent setup failed — you can retry later with: ./infra/scripts/setup-agent.ps1" -ForegroundColor Yellow - # } + # ── Step 1: Create AI agent ── + Write-Host "" + Write-Host "Setting up AI agent..." -ForegroundColor Yellow + & ./infra/scripts/setup-agent.ps1 + if ($LASTEXITCODE -ne 0) { + Write-Host "WARNING: Agent setup failed — you can retry later with: ./infra/scripts/setup-agent.ps1" -ForegroundColor Yellow + } # ── Step 1.5: Grant API identity access to SQL ── Write-Host "" @@ -103,9 +103,9 @@ hooks: python -m pip install --quiet --upgrade pip pip install --quiet -r requirements.txt - # echo "" - # echo "Setting up AI agent..." - # pwsh -File ./infra/scripts/setup-agent.ps1 || echo "WARNING: Agent setup failed — retry with: ./infra/scripts/setup-agent.ps1" + echo "" + echo "Setting up AI agent..." + pwsh -File ./infra/scripts/setup-agent.ps1 || echo "WARNING: Agent setup failed — retry with: ./infra/scripts/setup-agent.ps1" echo "" echo "Granting API identity SQL access..." diff --git a/src/api/modules/ingestion/azure_storage.py b/src/api/modules/ingestion/azure_storage.py index ec01ca01b..96da14f91 100644 --- a/src/api/modules/ingestion/azure_storage.py +++ b/src/api/modules/ingestion/azure_storage.py @@ -345,10 +345,11 @@ def delete_file_data(self, file_id: str, filename: str, doc_ids: list[str]) -> d except Exception: pass - # Delete doc-level blobs (documents/{doc_id}.json) + # Delete doc-level blobs ({doc_id}.json at container root, + # matching the path used by upload_to_blob) for doc_id in doc_ids: try: - blob = container.get_blob_client(f"documents/{doc_id}.json") + blob = container.get_blob_client(f"{doc_id}.json") blob.delete_blob() except Exception: pass @@ -387,5 +388,32 @@ def delete_file_data(self, file_id: str, filename: str, doc_ids: list[str]) -> d logger.info(f"Cleanup for file '{file_id}': {result}") return result + def clear_all_blobs(self) -> int: + """Delete every blob in the storage container (raw/, extracted/, documents/, etc.). + + Used when clearing all data before loading a new scenario so that stale + files from a previous scenario are not left behind in storage. + Returns the number of blobs deleted. + """ + settings = get_settings() + if not settings.azure_storage_account: + return 0 + + deleted = 0 + try: + blob_service = self._get_blob_client() + container = blob_service.get_container_client(settings.azure_storage_container) + blob_names = [b.name for b in container.list_blobs()] + for name in blob_names: + try: + container.delete_blob(name) + deleted += 1 + except Exception: + pass + logger.info(f"Cleared all blobs from container '{settings.azure_storage_container}': deleted {deleted}") + except Exception as e: + logger.warning(f"Failed to clear blobs: {e}") + return deleted + azure_storage_service = AzureStorageService() diff --git a/src/api/modules/ingestion/service.py b/src/api/modules/ingestion/service.py index af339d271..5dcb7b9b8 100644 --- a/src/api/modules/ingestion/service.py +++ b/src/api/modules/ingestion/service.py @@ -374,7 +374,7 @@ def _track_file(self, filename: str, data: list[dict]): """ from datetime import datetime - file_id = filename.rsplit(".", 1)[0] + file_id = filename.rsplit(".", 1)[0].replace(" ", "_") summary = "" keywords: list[str] = [] @@ -739,7 +739,7 @@ def load_json_file(self, file_path: str) -> IngestionResult: # Track doc_ids for reliable delete ingested_ids = [item["id"] for item in raw_data] from datetime import datetime - file_id = filename.rsplit(".", 1)[0] + file_id = filename.rsplit(".", 1)[0].replace(" ", "_") self._uploaded_files[file_id] = UploadedFile( id=file_id, filename=filename, @@ -778,7 +778,7 @@ def load_json_data(self, data: list[dict], filename: str = "uploaded_data.json") # Track file immediately (in-memory) so it appears in the file list right away from datetime import datetime - file_id = filename.rsplit(".", 1)[0] + file_id = filename.rsplit(".", 1)[0].replace(" ", "_") ingested_ids = [item["id"] for item in data] # Preserve existing status/error if file already exists (e.g., set to "processing" by upload) @@ -976,6 +976,14 @@ def clear(self): except Exception as e: logger.warning(f"Failed to purge queues: {e}") + # Delete all blobs (raw/, extracted/, documents/) so stale files from a + # previous scenario are not left behind in storage. + try: + from src.api.modules.ingestion.azure_storage import azure_storage_service + azure_storage_service.clear_all_blobs() + except Exception as e: + logger.warning(f"Failed to clear blobs: {e}") + def delete_file(self, file_id: str) -> bool: """Delete an uploaded file and all its documents.""" self._ensure_loaded() diff --git a/src/api/storage/sql_service.py b/src/api/storage/sql_service.py index 2c214ea15..bb3a05283 100644 --- a/src/api/storage/sql_service.py +++ b/src/api/storage/sql_service.py @@ -106,6 +106,7 @@ def _create_tables(self): summary NVARCHAR(MAX), keywords NVARCHAR(MAX), filter_values NVARCHAR(MAX), + doc_ids NVARCHAR(MAX), uploaded_at NVARCHAR(50) ) """) @@ -335,21 +336,23 @@ def save_uploaded_file(self, file_data: dict) -> bool: MERGE uploaded_files AS target USING (SELECT ? AS id) AS source ON target.id = source.id WHEN MATCHED THEN UPDATE SET - filename=?, doc_count=?, summary=?, keywords=?, filter_values=?, uploaded_at=? + filename=?, doc_count=?, summary=?, keywords=?, filter_values=?, doc_ids=?, uploaded_at=? WHEN NOT MATCHED THEN INSERT - (id, filename, doc_count, summary, keywords, filter_values, uploaded_at) - VALUES (?, ?, ?, ?, ?, ?, ?); + (id, filename, doc_count, summary, keywords, filter_values, doc_ids, uploaded_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?); """, file_data["id"], file_data.get("filename", ""), file_data.get("doc_count", 0), file_data.get("summary", ""), json.dumps(file_data.get("keywords", [])), json.dumps(file_data.get("filter_values", {})), + json.dumps(file_data.get("doc_ids", [])), file_data.get("uploaded_at", ""), # INSERT file_data["id"], file_data.get("filename", ""), file_data.get("doc_count", 0), file_data.get("summary", ""), json.dumps(file_data.get("keywords", [])), json.dumps(file_data.get("filter_values", {})), + json.dumps(file_data.get("doc_ids", [])), file_data.get("uploaded_at", ""), ) conn.commit() @@ -366,7 +369,7 @@ def load_all_uploaded_files(self) -> list[dict]: try: conn = self._get_connection() cursor = conn.cursor() - cursor.execute("SELECT id, filename, doc_count, summary, keywords, filter_values, uploaded_at FROM uploaded_files") + cursor.execute("SELECT id, filename, doc_count, summary, keywords, filter_values, doc_ids, uploaded_at FROM uploaded_files") rows = cursor.fetchall() conn.close() return [ @@ -375,7 +378,8 @@ def load_all_uploaded_files(self) -> list[dict]: "summary": r[3], "keywords": json.loads(r[4]) if r[4] else [], "filter_values": json.loads(r[5]) if r[5] else {}, - "uploaded_at": r[6] or "", + "doc_ids": json.loads(r[6]) if r[6] else [], + "uploaded_at": r[7] or "", } for r in rows ]