diff --git a/.github/workflows/deploy-sandbox.yml b/.github/workflows/deploy-sandbox.yml new file mode 100644 index 0000000..a04b59e --- /dev/null +++ b/.github/workflows/deploy-sandbox.yml @@ -0,0 +1,97 @@ +name: Deploy → sandbox + +# Triggers a Databricks bundle deploy (workflow + app) into the sandbox +# workspace, then triggers the discovery workflow once and gates promotion on +# the smoke_check_lakebase task. See docs/decisions/2026-06-06-cicd-deployment-pattern.md. +# +# This is the SANDBOX gate. Stage and prod deployments use sibling workflows +# with their own secrets and approval rules. + +on: + push: + branches: [main] + workflow_dispatch: {} + +permissions: + contents: read + +env: + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST_SANDBOX }} + DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID_SANDBOX }} + DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET_SANDBOX }} + BUNDLE_TARGET: dev + +jobs: + deploy-and-smoke: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: control-plane-app/package-lock.json + + - name: Install Databricks CLI + uses: databricks/setup-cli@main + + - name: Build frontend + working-directory: control-plane-app + run: | + npm ci + npm run build + + - name: Bundle validate + working-directory: workflows + run: databricks bundle validate --target "$BUNDLE_TARGET" + + - name: Bundle deploy + working-directory: workflows + run: databricks bundle deploy --target "$BUNDLE_TARGET" + + - name: Deploy app + working-directory: control-plane-app + run: bash deploy.sh + + - name: Trigger discovery workflow + id: trigger + run: | + # Find the deployed job ID from bundle state. + JOB_ID=$(databricks bundle summary --target "$BUNDLE_TARGET" --output json \ + --working-dir workflows \ + | python3 -c "import sys,json; d=json.load(sys.stdin); jobs=d.get('resources',{}).get('jobs',{}); print(next(iter(jobs.values())).get('id',''))") + if [ -z "$JOB_ID" ]; then + echo "Could not resolve job ID from bundle summary"; exit 1 + fi + echo "Triggering job $JOB_ID..." + RUN_ID=$(databricks jobs run-now "$JOB_ID" --output json | python3 -c "import sys,json; print(json.load(sys.stdin).get('run_id',''))") + echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" + echo "Triggered run $RUN_ID" + + - name: Wait for run + gate on smoke check + run: | + RUN_ID="${{ steps.trigger.outputs.run_id }}" + # Poll until terminal. The smoke task runs last in the DAG; if it fails, + # the whole run reports result_state=FAILED. + while true; do + STATE=$(databricks jobs get-run "$RUN_ID" --output json \ + | python3 -c "import sys,json; d=json.load(sys.stdin); s=d.get('state',{}); print(s.get('life_cycle_state','')+'/'+(s.get('result_state') or '-'))") + echo "$(date -u +%FT%TZ) state=$STATE" + case "$STATE" in + TERMINATED/SUCCESS) echo "✅ Discovery + smoke passed."; exit 0 ;; + TERMINATED/*|INTERNAL_ERROR/*|SKIPPED/*) + echo "❌ Discovery run terminal with non-success: $STATE" + # Surface per-task results to the CI log so the smoke FAIL message lands + databricks jobs get-run "$RUN_ID" --output json \ + | python3 -c " + import sys, json + d = json.load(sys.stdin) + for t in d.get('tasks', []): + s = t.get('state', {}) + print(f\" {t.get('task_key'):35s} {s.get('life_cycle_state'):12s} {s.get('result_state','-')}\")" + exit 1 ;; + *) sleep 30 ;; + esac + done diff --git a/.gitignore b/.gitignore index 44ba3c5..044683a 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ Thumbs.db # Local brainstorm / parking-lot notes (not part of the project) ideas/ +.claude/ +CLAUDE.md diff --git a/control-plane-app/backend/services/tools_service.py b/control-plane-app/backend/services/tools_service.py index 0faf272..1125429 100644 --- a/control-plane-app/backend/services/tools_service.py +++ b/control-plane-app/backend/services/tools_service.py @@ -53,8 +53,8 @@ def ensure_tools_tables(): for stmt in ddl_statements: try: execute_update(stmt) - except Exception as exc: - logger.warning("Tools DDL warning: %s", exc) + except Exception: + logger.exception("Tools DDL failed (statement=%s)", stmt[:80]) logger.info("Tools tables ensured") @@ -451,11 +451,17 @@ def refresh_tools(): _refresh_in_progress = True logger.info("Starting tools discovery …") + # Self-heal: the table is normally created by workflows/02_sync_to_lakebase + # Phase 7, but calling this defensively here means the Tools page works + # even if the workflow hasn't run yet (fresh deploy) or if the app SP + # somehow lost its read-only access to that table at boot. + ensure_tools_tables() + # Clear old MCP entries (they may be stale serving-endpoint records) try: execute_update("DELETE FROM tool_registry WHERE type = 'mcp_server'") - except Exception: - pass + except Exception as exc: + logger.warning("DELETE FROM tool_registry failed: %s", exc) # 1) Managed MCP — UC connections with is_mcp_connection mcp_conns = _discover_mcp_connections() @@ -481,8 +487,12 @@ def refresh_tools(): logger.info(" → UC functions: %s", len(funcs)) logger.info("Tools discovery complete") - except Exception as exc: - logger.warning("Tools refresh failed: %s", exc) + except Exception: + # Log with traceback. Returning silently here is what hid the + # "tool_registry does not exist" failure observed during onboarding — + # /api/v1/tools/sync answered 200 while the underlying refresh + # blew up on the missing table. + logger.exception("Tools refresh failed") finally: _refresh_in_progress = False _refresh_lock.release() diff --git a/control-plane-app/deploy.sh b/control-plane-app/deploy.sh index 1028cf1..a9335c7 100755 --- a/control-plane-app/deploy.sh +++ b/control-plane-app/deploy.sh @@ -134,18 +134,10 @@ echo "Deployment triggered. Monitor with:" echo " databricks apps get $APP_NAME $PROFILE_FLAG" # ── Grant the app SP access to Lakebase ─────────────────────── -# Idempotent — safe to re-run. Needs psycopg2 and databricks-sdk locally. +# Some workspaces block public Lakebase Postgres connectivity from the laptop +# ("Public access is not allowed for workspace ..."). When this happens, run +# the equivalent grant inside the workspace as a one-shot job instead — see +# run_grant_sp_lakebase_job.sh in this directory. echo "" -echo "Granting app SP access to Lakebase ..." -PROFILE_ENV="" -if [[ -n "$PROFILE_FLAG" ]]; then - PROFILE_ENV="DATABRICKS_CONFIG_PROFILE=${PROFILE_FLAG#--profile }" -fi -env $PROFILE_ENV \ - APP_NAME="$APP_NAME" \ - LAKEBASE_DNS="$LAKEBASE_DNS" \ - LAKEBASE_DATABASE="$LAKEBASE_DATABASE" \ - LAKEBASE_ENDPOINT_PATH="${LAKEBASE_ENDPOINT_PATH:-}" \ - LAKEBASE_INSTANCE="${LAKEBASE_INSTANCE:-}" \ - python3 grant_sp_lakebase.py \ - || echo " ⚠ grant_sp_lakebase.py failed — you may need to run it manually. See docs/installation.md." +echo "Skipping local grant_sp_lakebase.py — run via in-workspace job:" +echo " bash run_grant_sp_lakebase_job.sh ${PROFILE_FLAG:-}" diff --git a/control-plane-app/grant_sp_lakebase_notebook.py b/control-plane-app/grant_sp_lakebase_notebook.py new file mode 100644 index 0000000..3e85e33 --- /dev/null +++ b/control-plane-app/grant_sp_lakebase_notebook.py @@ -0,0 +1,42 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC # grant_sp_lakebase wrapper +# MAGIC In-workspace runner for `grant_sp_lakebase.py` — reads widget params, sets +# MAGIC them in the environment, then exec's the original script. + +# COMMAND ---------- + +# MAGIC %pip install --upgrade "databricks-sdk>=0.40.0" psycopg2-binary requests +# MAGIC dbutils.library.restartPython() + +# COMMAND ---------- + +import os, runpy + +dbutils.widgets.text("app_name", "", "App name") +dbutils.widgets.text("lakebase_dns", "", "Lakebase DNS") +dbutils.widgets.text("lakebase_database", "", "Lakebase database") +dbutils.widgets.text("lakebase_instance", "", "Lakebase instance (Provisioned)") +dbutils.widgets.text("lakebase_endpoint_path", "", "Lakebase endpoint path (Autoscaling)") +dbutils.widgets.text("script_path", "", "Path to grant_sp_lakebase.py in /Workspace") + +os.environ["APP_NAME"] = dbutils.widgets.get("app_name") +os.environ["LAKEBASE_DNS"] = dbutils.widgets.get("lakebase_dns") +os.environ["LAKEBASE_DATABASE"] = dbutils.widgets.get("lakebase_database") +os.environ["LAKEBASE_INSTANCE"] = dbutils.widgets.get("lakebase_instance") +os.environ["LAKEBASE_ENDPOINT_PATH"] = dbutils.widgets.get("lakebase_endpoint_path") + +script_path = dbutils.widgets.get("script_path") +print(f"Running grant_sp_lakebase.py from {script_path}") +print(f" APP_NAME={os.environ['APP_NAME']}") +print(f" LAKEBASE_DNS={os.environ['LAKEBASE_DNS']}") +print(f" LAKEBASE_INSTANCE={os.environ['LAKEBASE_INSTANCE']}") + +try: + runpy.run_path(script_path, run_name="__main__") + print("grant_sp_lakebase: completed without raising") +except SystemExit as e: + code = e.code if e.code is not None else 0 + print(f"grant_sp_lakebase: exited with code {code}") + if code != 0: + raise RuntimeError(f"grant_sp_lakebase.py exited with non-zero code {code}") diff --git a/control-plane-app/run_grant_sp_lakebase_job.sh b/control-plane-app/run_grant_sp_lakebase_job.sh new file mode 100755 index 0000000..b0fb116 --- /dev/null +++ b/control-plane-app/run_grant_sp_lakebase_job.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Run grant_sp_lakebase.py as a one-shot Databricks job inside the workspace. +# +# Use this when the local machine cannot reach Lakebase Postgres directly +# (e.g. when the workspace blocks public PG access). The job runs on +# serverless compute as the deploying user, who is the Lakebase admin. +# +# Reads .env from the current directory for APP_NAME / LAKEBASE_* settings. +# Pass --profile to use a non-default Databricks CLI profile. + +set -euo pipefail + +PROFILE_FLAG="" +while [[ $# -gt 0 ]]; do + case $1 in + --profile) PROFILE_FLAG="--profile $2"; shift 2 ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +if [ ! -f .env ]; then + echo "Error: .env not found in $(pwd). Run this from control-plane-app/." + exit 1 +fi + +# Load .env +set -a +while IFS='=' read -r key value; do + [[ -z "$key" || "$key" =~ ^# ]] && continue + export "$key=$value" +done < .env +set +a + +: "${APP_NAME:?Set APP_NAME in .env}" +: "${LAKEBASE_DNS:?Set LAKEBASE_DNS in .env}" +: "${LAKEBASE_DATABASE:?Set LAKEBASE_DATABASE in .env}" +LAKEBASE_INSTANCE="${LAKEBASE_INSTANCE:-}" +LAKEBASE_ENDPOINT_PATH="${LAKEBASE_ENDPOINT_PATH:-}" + +DB="databricks" +WORKSPACE_USER=$($DB auth describe $PROFILE_FLAG 2>/dev/null | grep -i "user" | head -1 | awk '{print $NF}') +WORKSPACE_DIR="/Workspace/Users/${WORKSPACE_USER}/ai-control-plane/control-plane-app" +SCRIPT_PATH="${WORKSPACE_DIR}/grant_sp_lakebase.py" +NB_PATH="${WORKSPACE_DIR}/grant_sp_lakebase_notebook" + +echo "Uploading grant_sp_lakebase.py ..." +$DB workspace import "$SCRIPT_PATH" \ + --file grant_sp_lakebase.py \ + --format AUTO --overwrite $PROFILE_FLAG + +echo "Uploading grant_sp_lakebase_notebook.py ..." +$DB workspace import "$NB_PATH" \ + --file grant_sp_lakebase_notebook.py \ + --format SOURCE --language PYTHON --overwrite $PROFILE_FLAG + +JOB_JSON=$(cat <=0.40.0", "psycopg2-binary", "requests"] + } + } + ] +} +EOF +) + +echo "Submitting one-shot job ..." +RESULT=$($DB jobs submit --json "$JOB_JSON" $PROFILE_FLAG -o json) +echo "$RESULT" | python3 -c "import sys,json +d=json.load(sys.stdin) +print(' run_id:', d.get('run_id')) +print(' result:', d.get('state',{}).get('result_state')) +print(' life_cycle:', d.get('state',{}).get('life_cycle_state')) +print(' message:', d.get('state',{}).get('state_message',''))" + +RUN_ID=$(echo "$RESULT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('run_id',''))") + +echo "" +echo "Task output:" +TASK_RUN_ID=$($DB jobs get-run "$RUN_ID" $PROFILE_FLAG -o json 2>&1 | python3 -c "import sys,json +d=json.load(sys.stdin); ts=d.get('tasks',[]); print(ts[0].get('run_id','') if ts else '')") +$DB jobs get-run-output "$TASK_RUN_ID" $PROFILE_FLAG -o json 2>&1 | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) + if d.get('error'): print('--- error ---'); print(d['error']) + if d.get('error_trace'): + import re + cleaned = re.sub(r'\\x1b\\[[0-9;]*m','',d['error_trace']) + print('--- error_trace ---'); print(cleaned[-1500:]) + if d.get('logs'): print('--- logs ---'); print(d['logs'][-1500:]) + if d.get('notebook_output',{}).get('result'): print('--- notebook result ---'); print(d['notebook_output']['result']) +except Exception as e: print('parse err:', e) +" || true + +echo "" +echo "Run finished. Inspect with:" +echo " databricks jobs get-run $RUN_ID $PROFILE_FLAG" diff --git a/docs/decisions/2026-06-06-cicd-deployment-pattern.md b/docs/decisions/2026-06-06-cicd-deployment-pattern.md new file mode 100644 index 0000000..e55639a --- /dev/null +++ b/docs/decisions/2026-06-06-cicd-deployment-pattern.md @@ -0,0 +1,124 @@ +# ADR: CI/CD deployment pattern for production rollout + +**Status:** Proposed (2026-06-06) +**Related:** +- [Findings log: deploy non-trivial fixes](../rca/2026-06-06-deploy-non-trivial-fixes.md) — the four issues that motivated the smoke gate. +- [PR #18 — gateway DDL transaction fix](https://github.com/databricks-solutions/agent-control-plane/pull/18) — introduces the smoke check this ADR depends on. + +## Context + +We just landed an initial sandbox deployment of `agent-control-plane`. The deployment surfaced a regression class that the existing pipeline does not catch: the `02_sync_to_lakebase` Phase 6 transaction bug created Delta tables successfully, reported `result=SUCCESS`, but silently failed to materialise `gateway_usage_daily` / `gateway_usage_hourly` in Lakebase. The deployed app then logged a continuous stream of `UndefinedTable` errors and rendered the Gateway page empty. The job result and the dashboard state disagreed. + +The fix (PR #18) currently lives on a fork. To roll the same stack into production responsibly, we want every future update to clear two gates before reaching prod: build-time tests of the code, and post-deploy smoke checks of the actual data plane (Lakebase tables exist with rows, app endpoints return non-empty payloads). This ADR defines that deployment pattern. + +## Decision + +**Adopt a three-environment promotion pipeline (sandbox → stage → prod) with a Lakebase smoke gate as the post-deploy quality bar in every environment.** Drive promotion from a CI/CD platform (proposal: GitHub Actions) using Databricks Asset Bundles and the Databricks CLI. + +### Environment matrix + +| Environment | Workspace | Trigger | Promotion criteria | +|---|---|---|---| +| `sandbox` | Sandbox workspace | Every push to `main` of the prod-tracking fork | Smoke task passes; manual sign-off optional | +| `stage` | Stage workspace (TBD — request from platform team) | Manual dispatch after sandbox smoke is green for ≥24h | Smoke + Playwright regression both pass | +| `prod` | Production workspace (TBD — request from platform team) | Manual dispatch with two-person approval | Stage has been green for ≥24h; smoke + regression pass against prod after deploy | + +We do not propose merging to `databricks-solutions/agent-control-plane`'s `main` as the prod trigger. The upstream is shared with other customers. Each adopting team should track its own internal repo (or fork) and pull in upstream `main` periodically. + +### Pipeline shape + +For each environment: + +``` +[lint+unit] → [bundle deploy --target ] → [app deploy] + ↓ +[run discovery workflow once] + ↓ +[smoke_check_lakebase task] ← the gate + ↓ +[Playwright regression — stage/prod only] + ↓ +[mark deployment green; promote] +``` + +Key choice: **the smoke check runs as a task inside the discovery workflow, not as a separate CI step.** This keeps the regression test next to the data it checks (same identity, same network, same Lakebase credentials), which means the smoke check actually exercises the production data plane rather than a CI-only proxy of it. The CI job's role is to *trigger* the workflow run, *poll* for completion, and *fail* if the smoke task fails. The bundle config wires `smoke_check_lakebase` to depend on `sync_to_lakebase`, so the existing Databricks job machinery handles ordering, retries, and email notifications. + +### Smoke gate spec + +`workflows/10_smoke_check_lakebase.py` (introduced alongside this ADR) categorises every Lakebase table the app reads from into three buckets: + +| Bucket | Behaviour | +|---|---| +| `REQUIRED` | Must exist AND have ≥1 row. Failure raises and the workflow goes red. | +| `EXPECTED` | Must exist. 0 rows is allowed (legitimate in quiet workspaces) but logged at WARN. | +| `OPTIONAL` | App-managed (created lazily on user action). Existence not asserted. | + +The bucketing is a contract derived from the actual `FROM ` references in `control-plane-app/backend/services/`. When a service starts reading from a new table, the smoke contract must be updated alongside the service change — that pairing is the unit of review. + +This catches three regression classes that the existing job result does not: +1. **Silent CREATE skips** like the gateway DDL transaction bug. +2. **Sync-side query that returns 0 rows** for what should be a populated table (e.g., a cron schedule that doesn't trigger, a role missing SELECT on a system table). +3. **Schema drift** — a column rename in the sync code that produces an empty INSERT clause. + +### Why GitHub Actions specifically + +| Option | Pro | Con | +|---|---|---| +| **GitHub Actions** | Already authenticated for the fork; community-supported `databricks/setup-cli` action; secrets handled by GH; easy fork→main promotion gate via PR | Requires a GH-side OAuth secret per target workspace | +| Azure DevOps Pipelines | Aligns with ADO-centric tooling some adopters already use | Crosses repo boundaries (GH for source, ADO for CI); harder to configure with a public-fork PR model | +| Databricks Workflows-only | Self-contained, no external CI | Can't run `bundle deploy` from inside Databricks (chicken-and-egg); poor model for blocking promotion on cross-environment results | +| Run smoke ad-hoc on laptop | Zero infra | Easy to skip; no audit trail of what shipped where; doesn't scale beyond one engineer | + +GitHub Actions is the path of least resistance for the public-fork model and gives us a documented audit trail. + +### Secrets layout + +For each environment (sandbox/stage/prod), GH repo secrets: +- `DATABRICKS_HOST_` — workspace URL +- `DATABRICKS_CLIENT_ID_` — service-principal applicationId +- `DATABRICKS_CLIENT_SECRET_` — service-principal OAuth secret +- `DATABRICKS_BUNDLE_VAR_*` — workspace-specific bundle variables (lakebase_dns, instance, warehouse_id, account_id) + +The deploying service principal needs: +- Workspace admin (deploy app + workflow) +- Metastore admin / `CREATE CATALOG` on metastore (or scope catalogs to a metastore-managed parent the SP already owns) +- `CAN_USE` on the SQL warehouse the workflow uses +- `databricks_superuser` role in Lakebase (created automatically when the SP first connects via `generate_database_credential`) + +### Playwright regression in stage/prod + +The existing `tests/governance-workspace-filter.spec.ts` is the seed. Extend it to walk all primary tabs (Governance, Observability, Gateway, Knowledge Bases, User Analytics, Agents) and assert that each has at least one populated chart/table. Run headless with a service-principal-issued OAuth token for the test user. Skip in sandbox to keep iteration fast; require in stage and prod. + +## Alternatives considered + +**Run the smoke check as a CI-side script that hits the deployed app's `/api/v1/*/page-data` endpoints.** This would catch *frontend* rendering issues that pure Lakebase checks miss. Rejected as the *primary* gate because (a) endpoint behaviour depends on the signed-in user's UC grants, which makes "is this empty because of permissions or because of missing data?" hard to answer from CI; (b) the SSO loop is brittle in CI; (c) it's still on the table as a *secondary* gate (Playwright in stage/prod fills this role). + +**Promote via merge to upstream `main`.** Rejected — upstream is multi-tenant; coupling our release cadence to the upstream maintainers' is fragile. Pull from upstream weekly into the fork's `main`; promote from there. + +**One smoke notebook per page (gateway, governance, observability, …).** Rejected — over-engineered for the current surface area. One contract file in one notebook is auditable in one diff. We can split later if it grows past a screen. + +## Consequences + +**Positive:** +- Future regressions of the silent-empty-table class fail loud at the workflow level, before users see empty screens. +- A single artefact (the smoke notebook + its contract) is the source of truth for "what does the app expect to find in Lakebase." +- Promotion criteria are written down; releasing to prod stops being an "is it working?" judgement call and becomes a checklist. +- The bundle stays the source of truth for both *what* deploys (job + app) and *what counts as a successful deploy* (smoke task green). + +**Negative:** +- Adds ~30s to every discovery run (smoke task overhead). +- Smoke contract has to stay in sync with the backend's `FROM
` set. A new dashboard tab that queries a new table needs the contract updated in the same PR — easy to forget. +- Stage/prod environments don't exist yet; this ADR depends on the platform team provisioning them. +- The CI-side OAuth secrets need rotation discipline (proposal: 90-day rotation, tracked in `docs/operations.md` once written). + +**Code-level impact:** +- New file: `workflows/10_smoke_check_lakebase.py` (added in PR #18 follow-up). +- New task in `workflows/databricks.yml`: `smoke_check_lakebase`, depending on `sync_to_lakebase`. +- New: `.github/workflows/deploy-sandbox.yml`, `.github/workflows/deploy-stage.yml`, `.github/workflows/deploy-prod.yml` (sandbox first; stage/prod after platform provisions the workspaces). +- New: `docs/operations.md` covering rotation, SP onboarding, runbook for smoke failures (separate PR). + +## Trigger conditions for revisiting + +- The smoke contract drifts from reality more than once. Replace the manual contract with introspection (e.g., a generated list from the backend's SQL strings). +- Stage/prod workspaces materialise. Update the environment matrix with their actual workspace IDs and reassess the SP grant requirements. +- Frontend regressions slip through the backend smoke gate. Promote Playwright from "stage/prod only" to the sandbox path too. diff --git a/docs/rca/2026-06-06-deploy-non-trivial-fixes.md b/docs/rca/2026-06-06-deploy-non-trivial-fixes.md new file mode 100644 index 0000000..d6f6230 --- /dev/null +++ b/docs/rca/2026-06-06-deploy-non-trivial-fixes.md @@ -0,0 +1,350 @@ +# Findings log: Non-trivial fixes during initial sandbox deployment + +**Date:** 2026-06-04 → 2026-06-06 +**Status:** Open log — items listed are deployment-blocking issues that exceeded the README/`docs/installation.md` instructions and required code or process changes. +**Workspace:** Sandbox (workspace and account identifiers redacted) + +## Summary + +The `agent-control-plane` README + `docs/installation.md` describe the happy-path setup: workspace admin runs `databricks bundle deploy`, runs `deploy.sh`, and the app + workflow start serving. On this sandbox that path was blocked at four points by environmental constraints not covered in the docs. Each blocker was resolved with a code or process change captured below so the next deployer does not have to re-discover them. + +--- + +## 1. Workspace-admin alone is not sufficient — metastore admin / `CREATE CATALOG` is required + +**Symptom.** `databricks bundle deploy --target dev` fails on the first task that creates the Unity Catalog catalog: + +``` +PERMISSION_DENIED: User does not have CREATE CATALOG on Metastore 'primary' +``` + +The user (a workspace admin) is a member of the `admins` group. + +**Root cause.** Workspace admin is a *workspace-scoped* role; UC metastore admin is *account-level* and orthogonal. The README assumes the deployer has both but does not say so explicitly. + +**Fix.** Request metastore-admin elevation on a privileged identity (a dedicated admin account, not the day-to-day developer account). Confirm with: + +```sh +databricks api post /api/2.0/sql/statements -p --json \ + '{"warehouse_id":"","statement":"SELECT current_metastore() AS m, is_account_group_member(\"account admins\") AS is_acct_admin"}' +``` + +Then deploy with `-p `. **Documentation gap:** `docs/installation.md` should list `CREATE CATALOG ON METASTORE` (or metastore-admin) as a hard prerequisite alongside workspace-admin. + +--- + +## 2. SSO session reuse silently authenticates the wrong identity + +**Symptom.** `databricks auth login -p admin-profile --host ` returned "successfully authenticated" without prompting for password — it had silently reused the existing Microsoft 365 / corporate SSO browser session for the day-to-day developer account. Every subsequent CLI call ran as the developer account, *not* the admin account, despite the profile name. + +**Root cause.** The Databricks browser-OAuth flow honours an existing IdP cookie. If the browser is already signed in to Microsoft 365 (or any IdP) with another work account, the OIDC step doesn't re-prompt. + +**Fix.** +1. Open an incognito window before the next `databricks auth login`. +2. After login, **always verify identity**: + ```sh + databricks current-user me -p -o json | jq .userName + ``` +3. If the wrong user surfaces, run `databricks auth logout -p ` and retry in a private session. + +**Documentation gap:** `docs/installation.md` should warn explicitly that profile names do not constrain identity — only the OAuth flow does — and add the `current-user me` verification step. + +--- + +## 3. Lakebase Postgres rejects laptop connections from this workspace + +**Symptom.** `deploy.sh` runs `grant_sp_lakebase.py` on the laptop, which connects to the Lakebase PG instance via psycopg2. Result: + +``` +psycopg2.OperationalError: External authorization failed. +Public access is not allowed for workspace . +``` + +This persists even on the corporate VPN. The Lakebase endpoint resolves to a public IP; the rejection is server-side, based on a workspace-attached Network Policy that disallows public ingress to Lakebase. + +**Root cause.** Workspace network policy blocks public PG access. There is no laptop-side fix — the connection has to originate from inside the workspace. + +**Fix.** Run the grant logic as a one-shot Databricks Job using serverless compute. New in this repo: + +| File | Role | +|---|---| +| `control-plane-app/grant_sp_lakebase_notebook.py` | Wrapper notebook: pip-installs deps, reads widget params, sets env vars, then `runpy.run_path`s `grant_sp_lakebase.py`. **Critical:** wraps in `try/except SystemExit` because the notebook framework treats `sys.exit(0)` (used by the script for normal completion) as a failure. | +| `control-plane-app/run_grant_sp_lakebase_job.sh` | Helper that uploads both files, submits a one-shot job (`notebook_task` + serverless `environments` spec with pinned `databricks-sdk>=0.40.0`, `psycopg2-binary`, `requests`). | +| `control-plane-app/deploy.sh` | Modified end-of-script: replaces the laptop-side `grant_sp_lakebase.py` invocation with a print pointing at `run_grant_sp_lakebase_job.sh`. | + +**Sub-traps encountered while building the in-workspace fallback:** + +- Submitting with `spark_python_task + environment_variables` failed: serverless `environments[].spec` does not honour `environment_variables`. Fixed by switching to `notebook_task` with `base_parameters` (widget reads). +- Default SDK on the serverless runtime did not have `databricks.sdk.service.postgres`. Fixed by pinning `databricks-sdk>=0.40.0` in the env spec **and** in a `%pip install --upgrade` cell at the top of the wrapper notebook. +- `sys.exit(0)` in `grant_sp_lakebase.py` reported as a task failure. Wrapper notebook catches `SystemExit` and only re-raises on non-zero exit codes. + +**Documentation gap:** `docs/installation.md` mentions Lakebase prerequisites but does not warn that workspaces with public-PG blocked require running the SP-grant step in-workspace. The new helper script + this RCA should be referenced from `docs/installation.md`. + +--- + +## 4. Sync workflow silently fails to create `gateway_usage_daily` and `gateway_usage_hourly` + +**Symptom.** After successful end-to-end deploy: +- `databricks apps logs ai-control-plane` shows repeated: + ``` + psycopg2.errors.UndefinedTable: relation "gateway_usage_daily" does not exist + ``` +- The Gateway page in the deployed app loads with empty metrics. +- Yet the `02_sync_to_lakebase` task reports `result=SUCCESS` on every recent run. +- The Delta source table `ai_control_plane.control_plane.gateway_usage_daily` exists and has rows. +- Direct in-workspace inspection of Lakebase `control_plane` confirms only the daily table is missing — every other expected table is present. + +**Root cause.** `workflows/02_sync_to_lakebase.py` Phase 6 (gateway sync) ran a single transaction containing CREATEs and ALTERs interleaved as: + +```python +[ + CREATE TABLE gateway_usage_daily (...), + CREATE INDEX idx_gud_date, + CREATE INDEX idx_gud_ep, + ALTER TABLE gateway_usage_daily ADD COLUMN IF NOT EXISTS rate_limited_count ..., + ALTER TABLE gateway_usage_hourly ADD COLUMN IF NOT EXISTS rate_limited_count ..., # <-- table doesn't exist yet + CREATE TABLE gateway_usage_hourly (...), +] +``` + +The `ALTER TABLE gateway_usage_hourly` raises `UndefinedTable`. **psycopg2 marks the entire transaction aborted** on any error; subsequent `cur.execute(ddl)` calls in the same transaction silently no-op until rollback. The per-statement `try/except` swallowed the warning print but did not rollback, so the trailing `CREATE TABLE gateway_usage_hourly` and the prior `CREATE TABLE gateway_usage_daily`'s commit went through partially — and on a fresh database, *neither* table exists when the connection commits. + +The observability section of the same file uses PostgreSQL `SAVEPOINT` per DDL, which contains failures correctly. The gateway section did not. + +**Fix.** `workflows/02_sync_to_lakebase.py` (Phase 6 DDL block): +1. Reordered DDLs so all `CREATE TABLE`s come before any `CREATE INDEX` / `ALTER TABLE`. +2. Switched from one shared transaction to one transaction per DDL: each statement runs in its own `cursor()` context with its own `commit()`, and a failure does `rollback()` and continues. This isolates the "ALTER on a not-yet-existing table on the very first run" case from poisoning sibling DDLs. + +**Verification step.** After the fix lands and the workflow re-runs, confirm tables exist with the in-workspace inspection notebook used during diagnosis (it queries `pg_tables` for the `control_plane` database via SDK-issued credentials): + +```python +# minimal inspector — submits as a notebook_task on serverless +from databricks.sdk import WorkspaceClient +import psycopg2 +w = WorkspaceClient() +me = w.current_user.me().user_name +token = w.database.generate_database_credential(instance_names=["ai-control-plane-db"]).token +conn = psycopg2.connect(host="", dbname="control_plane", + user=me, password=token, sslmode="require") +cur = conn.cursor() +cur.execute("SELECT tablename FROM pg_tables WHERE schemaname='public' " + "AND tablename LIKE 'gateway%'") +print(cur.fetchall()) +``` + +Expected: `[('gateway_inference_logs',), ('gateway_usage_daily',), ('gateway_usage_hourly',)]`. + +**Documentation gap:** No documentation gap — this is a code bug. Captured in this RCA so a future contributor sees the pattern (interleaved CREATE/ALTER + per-statement try/except + shared transaction = silent partial DDL). + +--- + +## Identity / permission audit (admin profile) + +For reference, what the privileged admin account had on the sandbox workspace as of 2026-06-06: + +| Layer | Status | Probe | +|---|---|---| +| Workspace admin | Yes — member of `admins` group | `current-user me` | +| Account admin | Yes — `is_account_admin=True` per app OBO log | App startup log | +| UC `SELECT` on `system.billing.usage` | Yes (43M rows in last 7d) | `/api/2.0/sql/statements` | +| UC `SELECT` on `system.serving.endpoint_usage` | Yes (71M rows / 7d) | same | +| UC `SELECT` on `system.ai_gateway.usage` | Yes (9M rows / 7d) | same | +| UC `SELECT` on `system.access.audit` | Yes (19B rows / 7d) | same | +| Lakebase `databricks_superuser` membership | Yes — confirmed via `pg_auth_members` | in-workspace inspector | +| SQL warehouse (Starter) | `CAN_MANAGE` via `admins` group | `warehouses get-permissions` | +| Account-level API (`/api/2.0/accounts/...`) | **No** — workspace OAuth profile returns `400 Unable to load OAuth Config` | direct curl | + +**Conclusion of the permission probe.** The "not all views populated" symptom that triggered this audit was *not* a permissions issue. Every UC system-table grant is in place. The actual bug is the Phase 6 DDL transaction pattern in `02_sync_to_lakebase.py` (item 4 above). The lack of account-level API access via the workspace-scoped OAuth profile is expected and only matters for operations like `account workspaces list` (not used by the deployed app at runtime). + +--- + +## Open follow-ups + +- [x] Re-deploy and verify gateway tables populate after the Phase 6 fix (done 2026-06-06; 21,303 daily / 2,606 hourly rows). +- [x] Roll the reusable changes into a PR ([#18](https://github.com/databricks-solutions/agent-control-plane/pull/18)). +- [x] Add a Lakebase smoke check task to the discovery workflow so this regression class fails loud instead of silently leaving the app empty (`workflows/10_smoke_check_lakebase.py`, wired as `smoke_check_lakebase` task in `databricks.yml`, depends on `sync_to_lakebase`). +- [x] Author CI/CD deployment-pattern ADR for the production rollout ([2026-06-06-cicd-deployment-pattern.md](../decisions/2026-06-06-cicd-deployment-pattern.md)). +- [x] Root-cause and fix the empty `billing_user_cost_daily` / `billing_product_daily` tables (item 5 — missing SELECT on `system.billing.list_prices` + silent-failure path in `09_discover_billing.py`). +- [x] Root-cause and fix the empty Tools page (item 6 — `tool_registry` table never created because the app SP doesn't have Lakebase DDL privs and the daemon-thread startup hook ran past its timeout; refresh helper swallowed all errors). +- [x] Document the `databricks bundle deploy` parameter-regression footgun (item 7 — placeholder defaults silently overwrite a working job; bit me while verifying item 6). +- [ ] Update `docs/installation.md` to reference items 1, 2, 3, the `system.billing` schema-level grant from item 5, and the `--var=` requirements from item 7. + +### New findings from the smoke check (2026-06-06) + +The first end-to-end smoke run on the sandbox surfaced **two `REQUIRED` tables that are empty post-sync** despite their upstream feeds being populated. Both have now been root-caused and fixed. + +## 5. Discovery silently writes 0 rows to all `system.billing.usage`-derived tables when the runtime identity lacks SELECT on `system.billing.list_prices` + +**Symptom.** After the gateway DDL fix in item 4 lands and the workflow re-runs successfully (`result=SUCCESS`), three Lakebase tables are still empty: + +- `billing_serving_daily` (0 rows in Lakebase) — drives Cost Overview by endpoint × SKU. +- `billing_product_daily` (0 rows in Lakebase) — drives the All Products breakdown. +- `billing_user_cost_daily` (0 rows in Lakebase) — drives per-user Endpoint Costs. + +The other two billing tables (`billing_token_daily`, `billing_user_endpoint_daily`) populate correctly. Token usage IS rendering on the Governance page; only the cost-related sections are blank. + +**Root cause.** `workflows/09_discover_billing.py` issues five SQL statements via `_execute_sql`. Three of them (queries 1, 3, 5) `LEFT JOIN system.billing.list_prices` to compute `total_cost_usd`. The deployer / workflow run-as identity had `SELECT ON SCHEMA system.billing` for *some* tables (granted ad-hoc) but **not** on `list_prices`. The JOIN fails with: + +``` +[INSUFFICIENT_PERMISSIONS] User does not have SELECT on Table 'system.billing.list_prices'. SQLSTATE: 42501 +``` + +The `_execute_sql` helper in `09_discover_billing.py` swallowed the failure: + +```python +if status != "SUCCEEDED": + err = resp.get("status", {}).get("error", {}) + print(f" SQL {status}: {err.get('message', '')[:300]}") + return [] # <-- silent zero-row path +``` + +That `return []` propagated as `serving_rows = 0`, which the writer happily persisted as a 0-row Delta table, which the sync task happily synced to a 0-row Lakebase table. The discovery task reports `result_state: SUCCESS` because the helper caught the API error before the notebook could fail. + +Critically, queries 2 and 4 (`token`, `user_endpoint`) hit `system.serving.endpoint_usage` only — no `list_prices` join, no permission gap, so they populated correctly. That's why token usage was showing while cost overviews were not. + +**Fix (two parts):** + +1. **Permission grant.** Granted `SELECT ON SCHEMA system.billing` to the discovery identity: + ```sql + GRANT SELECT ON SCHEMA system.billing TO `` + ``` + Verified via `SHOW GRANTS ON TABLE system.billing.list_prices`. After the grant, the same JOIN over the last 7 days returns 771,370 rows. + +2. **Make the silent path loud.** `workflows/09_discover_billing.py:170` — `_execute_sql` now `raise`s a `RuntimeError` with the API error code and message when the statement does not succeed, instead of returning `[]`. This converts future permission gaps from "0 rows in production with green CI" to "task FAILED with SQLSTATE in the error message". The smoke check would have caught this regardless, but failing the discovery task at the source is closer to the bug and gives a more actionable message than "EMPTY required table" three steps downstream. + +**Verification.** After the grant + helper fix re-deployed and the workflow re-ran (run `415862563272753`): + +| Table | Discovery rows | Lakebase rows | +|---|---:|---:| +| `billing_serving_daily` | 15,040 | 15,040 | +| `billing_product_daily` | 9,806 | 9,806 | +| `billing_user_cost_daily` | 4,472 | 4,472 | +| `billing_token_daily` | 3,932 | 3,932 | +| `billing_user_endpoint_daily` | 21,674 | 21,674 | + +**Documentation gap.** `docs/installation.md` should list `SELECT ON SCHEMA system.billing` (not just `system.billing.usage`) as the discovery-identity grant — this is the difference between "Governance has token counts" and "Governance has actual cost numbers". Also worth noting: the `LEFT JOIN system.billing.list_prices` predicate means cost discovery is *all-or-nothing* on the join — partial grants produce silent zeros, not partial data. + +--- + +## 6. Tools page renders empty / 500s when app SP lacks Lakebase DDL privileges + +**Symptom.** After the deployment is otherwise healthy (Governance tabs working, Gateway tabs working, agents discovered), the **Tools** section in the app is broken across all four tabs: + +- `/api/v1/tools/overview` — returns HTTP 500 `Internal Server Error` +- `/api/v1/tools/mcp-servers` — returns HTTP 500 +- `/api/v1/tools/functions` — returns HTTP 500 +- `/api/v1/tools/usage` — returns 200 with `[]` + +The frontend Tools tabs all render empty. + +The `/api/v1/tools/sync` endpoint (the only POST route on this router) returns HTTP 200 `{"status":"ok","message":"Tools refresh complete"}` regardless — it lies. + +**Root cause.** Three layered failures masked each other: + +1. **The `tool_registry` Lakebase table was never created.** App startup runs `_init_tools` in a daemon thread that calls `ensure_tools_tables()` (which `CREATE TABLE IF NOT EXISTS tool_registry ...`) followed by `maybe_refresh_async()`. The whole `_run_all_inits` fan-out is wrapped in `t.join(timeout=120)`, after which the server starts regardless. On this sandbox the daemon thread either crashed before `_init_tools` ran, ran past the 120 s budget, or hit a Lakebase auth issue under load — in all three cases the server boots without `tool_registry`. + +2. **`ensure_tools_tables()` swallowed real DDL failures.** Each statement was wrapped in `try/except: logger.warning(...)`, so even when the DDL truly failed (e.g. SP doesn't have `CREATE` on the schema), the only signal was a warning line that was buried among others and rolled out of the app's short log retention. + +3. **`refresh_tools()` swallowed the entire body.** The function was wrapped in `try/except Exception: logger.warning("Tools refresh failed: %s", exc)`. When `_upsert_tools` or `_discover_uc_functions` blew up against the missing table, the warning printed once and the lock released — and `/api/v1/tools/sync` happily returned 200. There was no traceback in logs and no error in the API response: a perfect silent break. + +The dashboard route `/api/v1/tools/overview` raised the underlying `psycopg2.errors.UndefinedTable: relation "tool_registry" does not exist` to FastAPI's default 500 handler. That was the only externally visible signal that anything was wrong, and it took five minutes of `grep -B3 -A3 "GET /api/v1/tools/overview HTTP/1.1\\\" 500"` against `databricks apps logs` to find the underlying cause. + +The same architectural issue exists for `request_logs` — it's another app-managed table created by the request-audit middleware, also fails silently if DDL isn't possible. + +**Fix (three parts):** + +1. **Move app-managed table DDL into the workflow.** Added Phase 7 to `workflows/02_sync_to_lakebase.py` that creates `tool_registry` and `request_logs` from the workflow run-as identity (which is `databricks_superuser` on the Lakebase PG instance). The app no longer needs DDL privileges to function — only `SELECT/INSERT/UPDATE/DELETE` on existing tables. This parallels the pattern for `discovered_agents`, `gateway_usage_*`, and 20+ other tables already created here. + +2. **Make `tools_service._refresh_tools` self-heal and stop swallowing errors.** + - `refresh_tools()` now calls `ensure_tools_tables()` at the top of its body, so if the workflow hasn't run yet (fresh deploy) the read path can recover on its own. + - The catch-all in `refresh_tools()` now uses `logger.exception(...)` instead of `logger.warning(...)`, giving a full traceback when something fails. + - Per-statement exception handlers in `ensure_tools_tables()` now use `logger.exception(...)` with the offending DDL statement included. + +3. **Add `tool_registry` and `request_logs` to the smoke check.** Both are now in the `EXPECTED` bucket of `workflows/10_smoke_check_lakebase.py` — existence is asserted on every workflow run; 0 rows is a WARN not a failure (legitimate before any user activity), but the table being missing now flips `result_state` to FAILED with an actionable message. + +**Why "EXPECTED" rather than "REQUIRED" for these.** Both tables are populated by user/app activity (the SP discovers MCP servers + UC functions on first request; request_logs accumulates as users interact with the app). On a freshly deployed sandbox with no activity yet, 0 rows is correct. The failure mode we're guarding against is "table missing entirely", which now correctly fires a smoke-check error instead of a 500 in the dashboard. + +**Verification.** After the workflow re-ran with the Phase 7 patch (run `881046971716582`): + +```sh +$ curl -sS -H "Authorization: Bearer $TOKEN" \ + https://.databricksapps.com/api/v1/tools/overview +{"total_tools":3,"mcp_servers":3,"uc_functions":0,"managed_count":3, + "custom_app_count":0,"is_refreshing":false,"last_refreshed":"2026-06-06T..."} +``` + +| Tab | Before | After | +|---|---|---| +| Overview | HTTP 500 (UndefinedTable: tool_registry) | 3 MCP servers, 0 UC functions | +| MCP Servers | HTTP 500 | 3 entries: Atlassian / Google Drive / SharePoint (system-managed) | +| UC Functions | HTTP 500 | Empty (legitimate — no functions in `ai_control_plane.control_plane`; SP also only sees 6 catalogs) | +| Usage | `[]` (worked because no Lakebase touch) | `[]` (legitimate — no MLflow traces with TOOL/FUNCTION spans yet) | + +The remaining "empty" UC Functions and Usage tabs are correct given the current sandbox state — there are zero UC functions in the project's catalog, and no MLflow traces with tool spans. Both will populate naturally as agents using UC function tools are deployed. + +**Documentation gap.** `docs/installation.md` does not currently call out that the app SP needs read access to `ai_control_plane.control_plane`'s UC functions (and any other catalogs the deployer wants surfaced in the Tools tab). This is a workspace-by-workspace concern (catalog grants are deployer-discretion); should be flagged in the Tools page as "no functions found — grant `USE CATALOG` to the app SP" rather than silently empty. + +--- + +## 7. `databricks bundle deploy` without explicit `--var` flags actively breaks a working job + +**Symptom.** While verifying the item-6 fix, ran `databricks bundle deploy --target dev -p ` from the `workflows/` directory to register the newly-added `smoke_check_lakebase` task. The CLI reported `Deployment complete!`. The next workflow run then failed every task with: + +``` +[PARSE_SYNTAX_ERROR] Syntax error at or near '<'. SQLSTATE: 42601 +spark.sql(f"CREATE SCHEMA IF NOT EXISTS {CATALOG}.{SCHEMA}") + ^^^^^^^^^^^^^^^^ +``` + +Inspecting the deployed job revealed `base_parameters` had been overwritten with the literal placeholder strings: + +```json +"base_parameters": { + "billing_retention_days": "90", + "catalog": "", + "schema": "control_plane", + "warehouse_id": "" +} +``` + +**Root cause.** `workflows/databricks.yml` defines the `dev` target with template-style placeholder defaults: + +```yaml +targets: + dev: + mode: development + default: true + variables: + catalog: # e.g. main, my_catalog + lakebase_dns: # e.g. ep-xxxx.database.cloud.databricks.com + warehouse_id: # SQL warehouse ID + account_id: # Databricks account ID +``` + +These are not Bundle interpolation tokens; they are literal string defaults. A subsequent `databricks bundle deploy --target dev` without `--var=` overrides happily writes those literal values into the running job's parameters. The CLI does not warn that you are about to overwrite a working configuration with a placeholder. + +The previous (working) deploy was done by someone who knew to pass `--var="catalog=ai_control_plane" --var="warehouse_id=..."`. There is no record of those values in the repo and no `.databricks-bundle.local.yml` overlay — the parameters were ephemeral to that one CLI invocation. + +**Fix (immediate).** Re-deploy with all variables supplied: + +```sh +cd workflows +databricks bundle deploy --target dev -p \ + --var="catalog=" \ + --var="schema=control_plane" \ + --var="lakebase_dns=" \ + --var="lakebase_endpoint_path=" \ + --var="lakebase_instance=" \ + --var="warehouse_id=" \ + --var="account_id=" +``` + +Confirmed end-to-end (run `430345618113642`): all 11 tasks SUCCESS, smoke check passed with 12/12 REQUIRED tables OK and `tool_registry` + `request_logs` present in the EXPECTED bucket. + +**Fix (durable).** Two complementary changes worth doing in a follow-up PR: + +1. Replace the `` placeholder defaults with values that *fail loud* if not overridden — e.g. `default: __UNSET__` plus a CI-time check in `deploy.sh` / the GitHub Actions workflow that grep-rejects any value matching `__UNSET__` after rendering. The current behaviour silently overwrites prod-like configurations. + +2. Persist target-specific values to a `.databricks-bundle..local.yml` (gitignored) overlay loaded automatically by the CLI, so a `databricks bundle deploy` from any workstation without explicit `--var=` flags is either a no-op (correctly resolved from the overlay) or fails fast on missing overlay. + +**Documentation gap.** `docs/installation.md` should explicitly enumerate the required `--var=` flags for `databricks bundle deploy`, OR direct deployers to author a local overlay file with the values pinned. Right now a fresh deployer who follows the README steps will succeed *only if* they happen to pass the right flags — and a successful re-deploy by someone who has those flags is actively destructive to a job configured by anyone who didn't. diff --git a/workflows/02_sync_to_lakebase.py b/workflows/02_sync_to_lakebase.py index a745800..3ab19ff 100644 --- a/workflows/02_sync_to_lakebase.py +++ b/workflows/02_sync_to_lakebase.py @@ -1412,42 +1412,47 @@ def _parse_gateway_payload(req_raw: str, resp_raw: str) -> dict: gw_daily_count = 0 gw_hourly_count = 0 -with gw_conn.cursor() as cur: - for ddl in [ - """CREATE TABLE IF NOT EXISTS gateway_usage_daily ( - usage_date DATE NOT NULL, - endpoint_name TEXT NOT NULL, - requester TEXT DEFAULT '', - request_count BIGINT DEFAULT 0, - input_tokens BIGINT DEFAULT 0, - output_tokens BIGINT DEFAULT 0, - error_count BIGINT DEFAULT 0, - rate_limited_count BIGINT DEFAULT 0, - last_synced TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - PRIMARY KEY (usage_date, endpoint_name, requester) - )""", - "CREATE INDEX IF NOT EXISTS idx_gud_date ON gateway_usage_daily (usage_date DESC)", - "CREATE INDEX IF NOT EXISTS idx_gud_ep ON gateway_usage_daily (endpoint_name)", - # ADD COLUMN for tables that pre-existed without rate_limited_count - "ALTER TABLE gateway_usage_daily ADD COLUMN IF NOT EXISTS rate_limited_count BIGINT DEFAULT 0", - "ALTER TABLE gateway_usage_hourly ADD COLUMN IF NOT EXISTS rate_limited_count BIGINT DEFAULT 0", - """CREATE TABLE IF NOT EXISTS gateway_usage_hourly ( - hour TEXT NOT NULL, - endpoint_name TEXT DEFAULT '', - request_count BIGINT DEFAULT 0, - input_tokens BIGINT DEFAULT 0, - output_tokens BIGINT DEFAULT 0, - error_count BIGINT DEFAULT 0, - rate_limited_count BIGINT DEFAULT 0, - last_synced TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - PRIMARY KEY (hour, endpoint_name) - )""", - ]: - try: +_gw_ddls = [ + """CREATE TABLE IF NOT EXISTS gateway_usage_daily ( + usage_date DATE NOT NULL, + endpoint_name TEXT NOT NULL, + requester TEXT DEFAULT '', + request_count BIGINT DEFAULT 0, + input_tokens BIGINT DEFAULT 0, + output_tokens BIGINT DEFAULT 0, + error_count BIGINT DEFAULT 0, + rate_limited_count BIGINT DEFAULT 0, + last_synced TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + PRIMARY KEY (usage_date, endpoint_name, requester) + )""", + """CREATE TABLE IF NOT EXISTS gateway_usage_hourly ( + hour TEXT NOT NULL, + endpoint_name TEXT DEFAULT '', + request_count BIGINT DEFAULT 0, + input_tokens BIGINT DEFAULT 0, + output_tokens BIGINT DEFAULT 0, + error_count BIGINT DEFAULT 0, + rate_limited_count BIGINT DEFAULT 0, + last_synced TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + PRIMARY KEY (hour, endpoint_name) + )""", + "CREATE INDEX IF NOT EXISTS idx_gud_date ON gateway_usage_daily (usage_date DESC)", + "CREATE INDEX IF NOT EXISTS idx_gud_ep ON gateway_usage_daily (endpoint_name)", + # ADD COLUMN for tables that pre-existed without rate_limited_count + "ALTER TABLE gateway_usage_daily ADD COLUMN IF NOT EXISTS rate_limited_count BIGINT DEFAULT 0", + "ALTER TABLE gateway_usage_hourly ADD COLUMN IF NOT EXISTS rate_limited_count BIGINT DEFAULT 0", +] +# Run each DDL in its own transaction so a single failure cannot abort the +# whole batch (psycopg2 marks the transaction aborted on any error and +# silently no-ops subsequent statements until rollback). +for ddl in _gw_ddls: + try: + with gw_conn.cursor() as cur: cur.execute(ddl) - except Exception as e: - print(f" DDL warning: {e}") - gw_conn.commit() + gw_conn.commit() + except Exception as e: + gw_conn.rollback() + print(f" DDL warning: {e}") # COMMAND ---------- @@ -1910,6 +1915,77 @@ def _stamp_cache_meta(conn, cache_key: str, rows_loaded: int) -> None: # COMMAND ---------- +# MAGIC %md +# MAGIC ## Phase 7: ensure app-managed tables +# MAGIC +# MAGIC `tool_registry` and `request_logs` are populated lazily by the deployed +# MAGIC app (Tools page discovery + per-request audit logging). The app's +# MAGIC startup hook tries to `CREATE TABLE IF NOT EXISTS` these on first boot, +# MAGIC but the app's service principal does not always have Lakebase DDL +# MAGIC privileges — when it doesn't, the failure is swallowed in a daemon +# MAGIC thread and the Tools page renders empty with a 500 from `/tools/overview`. +# MAGIC +# MAGIC The workflow run-as identity is `databricks_superuser`, so DDL here is +# MAGIC always safe. Creating the tables upfront makes the app independent of +# MAGIC its own DDL grants. + +# COMMAND ---------- + +print("\n" + "═" * 70) +print("Phase 7: app-managed tables (tool_registry, request_logs)") +print("═" * 70) + +_app_ddls = [ + """CREATE TABLE IF NOT EXISTS tool_registry ( + tool_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + type TEXT NOT NULL, + sub_type TEXT, + endpoint_name TEXT, + catalog_name TEXT, + schema_name TEXT, + description TEXT, + status TEXT, + config JSONB, + last_synced TIMESTAMP WITH TIME ZONE DEFAULT NOW() + )""", + "CREATE INDEX IF NOT EXISTS idx_tr_type ON tool_registry (type)", + """CREATE TABLE IF NOT EXISTS request_logs ( + request_id TEXT PRIMARY KEY, + agent_id TEXT, + model_id TEXT, + user_id TEXT, + timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + query_text TEXT, + response_text TEXT, + latency_ms NUMERIC(12,2), + status_code INTEGER, + input_tokens INTEGER DEFAULT 0, + output_tokens INTEGER DEFAULT 0, + cost_usd NUMERIC(12,6) DEFAULT 0, + error_message TEXT, + endpoint_type TEXT + )""", + "CREATE INDEX IF NOT EXISTS idx_rl_agent ON request_logs (agent_id)", + "CREATE INDEX IF NOT EXISTS idx_rl_ts ON request_logs (timestamp DESC)", + "CREATE INDEX IF NOT EXISTS idx_rl_user ON request_logs (user_id)", +] + +app_conn = get_lakebase_connection() +for ddl in _app_ddls: + try: + with app_conn.cursor() as cur: + cur.execute(ddl) + app_conn.commit() + print(f" ✅ {ddl.split()[2:5]}") + except Exception as e: + app_conn.rollback() + print(f" ⚠️ DDL warning: {e}") +app_conn.close() +print(" Phase 7 complete") + +# COMMAND ---------- + # MAGIC %md # MAGIC ## Final Summary diff --git a/workflows/09_discover_billing.py b/workflows/09_discover_billing.py index 305ded9..5555d0d 100644 --- a/workflows/09_discover_billing.py +++ b/workflows/09_discover_billing.py @@ -170,8 +170,7 @@ def _execute_sql(sql: str) -> List[Dict[str, Any]]: try: resp = w.api_client.do("POST", "/api/2.0/sql/statements", body=body) except Exception as exc: - print(f" SQL failed: {exc}") - return [] + raise RuntimeError(f"SQL submission failed: {exc}") from exc status = resp.get("status", {}).get("state", "") sid = resp.get("statement_id", "") if status in ("PENDING", "RUNNING") and sid: @@ -186,8 +185,13 @@ def _execute_sql(sql: str) -> List[Dict[str, Any]]: break if status != "SUCCEEDED": err = resp.get("status", {}).get("error", {}) - print(f" SQL {status}: {err.get('message', '')[:300]}") - return [] + # Promote to a task failure. Returning [] silently here is what masked + # the missing SELECT on system.billing.list_prices for weeks — every + # billing_*_daily Lakebase table downstream of these queries was + # written empty and the discovery task still reported SUCCESS. + raise RuntimeError( + f"SQL {status}: {err.get('error_code','')} {err.get('message','')[:500]}" + ) cols = [c["name"] for c in resp.get("manifest", {}).get("schema", {}).get("columns", [])] total_chunks = int(resp.get("manifest", {}).get("total_chunk_count") or 0) diff --git a/workflows/10_smoke_check_lakebase.py b/workflows/10_smoke_check_lakebase.py new file mode 100644 index 0000000..3c64582 --- /dev/null +++ b/workflows/10_smoke_check_lakebase.py @@ -0,0 +1,292 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC # Lakebase smoke check +# MAGIC +# MAGIC Final task in the discovery workflow. Connects to Lakebase and asserts +# MAGIC every table the deployed app reads from exists. Surfaces row counts so a +# MAGIC silent-empty-table failure (like the gateway_usage_daily transaction +# MAGIC bug) shows up here as a workflow FAILED instead of as empty UI screens. +# MAGIC +# MAGIC Three categories: +# MAGIC +# MAGIC | Category | Behaviour | +# MAGIC |-------------|--------------------------------------------------------| +# MAGIC | `required` | Must exist AND have ≥1 row. Failure raises. | +# MAGIC | `expected` | Must exist. 0 rows is allowed (quiet workspace) but | +# MAGIC | | logged at WARN. Doc gap if perpetually empty. | +# MAGIC | `optional` | App-managed; created on first user action. Existence | +# MAGIC | | NOT required. | + +# COMMAND ---------- + +# MAGIC %pip install psycopg2-binary "databricks-sdk>=0.40.0" --upgrade +# MAGIC dbutils.library.restartPython() + +# COMMAND ---------- + +import json +import psycopg2 +from databricks.sdk import WorkspaceClient + +dbutils.widgets.text("lakebase_dns", "", "Lakebase host (DNS)") +dbutils.widgets.text("lakebase_database", "control_plane", "Lakebase database") +dbutils.widgets.text("lakebase_instance", "", "Lakebase instance name (Provisioned only)") +dbutils.widgets.text("lakebase_endpoint_path", "", "Lakebase endpoint path (Autoscaling only)") + +DNS = dbutils.widgets.get("lakebase_dns") +DB = dbutils.widgets.get("lakebase_database") +INSTANCE = dbutils.widgets.get("lakebase_instance") +ENDPOINT_PATH = dbutils.widgets.get("lakebase_endpoint_path") + +if not DNS: + raise ValueError("lakebase_dns must be set via job parameters") +if not INSTANCE and not ENDPOINT_PATH: + raise ValueError("Either lakebase_instance or lakebase_endpoint_path must be set") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Table contract +# MAGIC +# MAGIC Source: every `FROM
` reference in `control-plane-app/backend/services/`. +# MAGIC Required tables are those the dashboard relies on for non-empty rendering of a +# MAGIC primary KPI. Expected tables are populated by discovery but may be 0 in a quiet +# MAGIC workspace. Optional tables are app-managed (created lazily on user action). + +# COMMAND ---------- + +REQUIRED = [ + # Sync workflow → Delta → Lakebase. If any of these are missing or empty + # post-sync, a primary dashboard tab WILL render empty and we want CI red. + "discovered_agents", + "billing_user_endpoint_daily", + "billing_user_cost_daily", + "billing_token_daily", + "billing_product_daily", + "gateway_usage_daily", + "gateway_usage_hourly", + "user_analytics_daily", + "user_analytics_heatmap", + "lakebase_instances", + "uag_usage_summary", + "uag_usage_breakdown", +] + +EXPECTED = [ + # Populated by discovery but legitimately empty in some workspaces. + # We assert existence; row count of 0 produces a WARN, not a failure. + "billing_serving_daily", # may be empty if no model serving in last 90d + "observability_traces", + "observability_trace_details", + "observability_experiments", + "observability_runs", + "gateway_inference_logs", # only populated when Mosaic AI Gateway inference logging is enabled + "vector_search_endpoints", # workspace may have no Vector Search endpoints + "vector_search_indexes", # endpoints can exist with 0 indexes + "vector_search_health_history", + "kb_billing_daily", # only populated when Vector Search indexes exist + "billing_cache_meta", + # App-managed tables created in Phase 7 of 02_sync_to_lakebase. These were + # historically created lazily by the app's startup hook, but the app SP + # does not always have Lakebase DDL privileges — when it doesn't, the + # Tools page rendered empty with a 500 from /api/tools/overview while the + # job still reported SUCCESS. Asserted as EXPECTED (existence required, + # 0 rows allowed before first user activity). + "tool_registry", + "request_logs", +] + +OPTIONAL = [ + # Created lazily by the app on first user action. Don't assert existence. + "playground_sessions", + "playground_messages", + "workspace_registry", + "agent_permissions_cache", + "agent_registry", +] + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Connect + +# COMMAND ---------- + +w = WorkspaceClient() +me = w.current_user.me().user_name + +if INSTANCE: + creds = w.database.generate_database_credential(instance_names=[INSTANCE]) + token = creds.token +else: + # Autoscaling endpoint path + import requests + auth_h = {} + w.config.authenticate(auth_h) + resp = requests.post( + f"{w.config.host.rstrip('/')}/api/2.0/postgres/credentials", + headers={**auth_h, "Content-Type": "application/json"}, + json={"endpoint": ENDPOINT_PATH}, + timeout=20, + ) + resp.raise_for_status() + token = resp.json().get("token", "") + +print(f"Connecting to Lakebase {DNS}/{DB} as {me}") + +conn = psycopg2.connect( + host=DNS, port=5432, dbname=DB, user=me, password=token, + sslmode="require", connect_timeout=15, +) +conn.autocommit = True + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Run checks + +# COMMAND ---------- + +def list_user_tables(cur) -> set: + cur.execute( + "SELECT tablename FROM pg_tables " + "WHERE schemaname = 'public'" + ) + return {r[0] for r in cur.fetchall()} + +def safe_count(cur, table: str): + try: + cur.execute(f'SELECT COUNT(*) FROM "{table}"') + return cur.fetchone()[0], None + except psycopg2.Error as exc: + return None, str(exc).strip() + +results = {"required": [], "expected": [], "optional": []} +failures = [] +warnings = [] + +with conn.cursor() as cur: + present = list_user_tables(cur) + + for t in REQUIRED: + if t not in present: + failures.append(f"MISSING required table: {t}") + results["required"].append({"table": t, "status": "MISSING", "rows": None}) + continue + rows, err = safe_count(cur, t) + if err: + failures.append(f"COUNT failed on required table {t}: {err}") + results["required"].append({"table": t, "status": "ERROR", "rows": None, "error": err}) + continue + if rows == 0: + failures.append(f"EMPTY required table: {t}") + results["required"].append({"table": t, "status": "EMPTY", "rows": 0}) + else: + results["required"].append({"table": t, "status": "OK", "rows": rows}) + + for t in EXPECTED: + if t not in present: + failures.append(f"MISSING expected table: {t}") + results["expected"].append({"table": t, "status": "MISSING", "rows": None}) + continue + rows, err = safe_count(cur, t) + if err: + warnings.append(f"COUNT failed on expected table {t}: {err}") + results["expected"].append({"table": t, "status": "ERROR", "rows": None, "error": err}) + continue + if rows == 0: + warnings.append(f"empty expected table: {t}") + results["expected"].append({"table": t, "status": "EMPTY", "rows": 0}) + else: + results["expected"].append({"table": t, "status": "OK", "rows": rows}) + + for t in OPTIONAL: + if t not in present: + results["optional"].append({"table": t, "status": "ABSENT", "rows": None}) + continue + rows, err = safe_count(cur, t) + results["optional"].append({ + "table": t, + "status": "OK" if not err else "ERROR", + "rows": rows, + **({"error": err} if err else {}), + }) + +conn.close() + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Report + +# COMMAND ---------- + +print("=" * 70) +print("Lakebase smoke check report") +print("=" * 70) + +for category in ["required", "expected", "optional"]: + print(f"\n### {category.upper()} ({len(results[category])} tables)") + for r in sorted(results[category], key=lambda x: (x["status"] != "OK", x["table"])): + rows = r.get("rows") + rows_s = f"{rows:>10,}" if isinstance(rows, int) else f"{'-':>10}" + marker = { + "OK": "✅", + "EMPTY": "⚠️ ", + "MISSING": "❌", + "ERROR": "❌", + "ABSENT": "· ", + }.get(r["status"], "? ") + line = f" {marker} {r['table']:<40s} {rows_s} rows ({r['status']})" + if r.get("error"): + line += f" [{r['error'][:80]}]" + print(line) + +print("\n" + "=" * 70) +if warnings: + print(f"WARNINGS ({len(warnings)}):") + for w_ in warnings: + print(f" ⚠️ {w_}") + +summary = { + "status": "FAIL" if failures else "PASS", + "failures": failures, + "warnings": warnings, + "results": results, +} +# Always print the JSON so it shows up in task logs even when we raise. +print("\nSMOKE_RESULT_JSON:", json.dumps(summary)) + +if failures: + print(f"\nFAILURES ({len(failures)}):") + for f_ in failures: + print(f" ❌ {f_}") + print("=" * 70) + # Raise — this propagates as a task failure, which is what we want CI to gate on. + # We deliberately do NOT call dbutils.notebook.exit() here: that would mark the + # task SUCCEEDED with the JSON as its return value, hiding the FAIL from the + # workflow-level result_state. + # + # Inline the failure list and a tight per-table breakdown into the exception + # message because notebook stdout is not surfaced through `jobs get-run-output` + # on failure — the message body is the only post-mortem channel that actually + # round-trips back to a CI runner via the API. + breakdown_lines = [] + for cat in ["required", "expected"]: + for r in results[cat]: + if r["status"] != "OK": + rows = r.get("rows") + rows_s = str(rows) if isinstance(rows, int) else "-" + breakdown_lines.append(f" [{cat}] {r['table']:<35s} {r['status']:<8s} rows={rows_s}") + raise RuntimeError( + "Lakebase smoke check FAILED: " + + "; ".join(failures) + + "\nBreakdown:\n" + + "\n".join(breakdown_lines) + ) + +print("\n✅ All required tables present and populated.") +if warnings: + print(f" ({len(warnings)} non-fatal warning(s) — see above)") +print("=" * 70) +dbutils.notebook.exit(json.dumps(summary)) diff --git a/workflows/databricks.yml b/workflows/databricks.yml index 4d77865..9ee973c 100644 --- a/workflows/databricks.yml +++ b/workflows/databricks.yml @@ -187,6 +187,19 @@ resources: warehouse_id: ${var.warehouse_id} environment_key: default + - task_key: smoke_check_lakebase + description: "Assert every Lakebase table the dashboard reads from exists with the expected row content. Fails the workflow if a required table is missing or empty (e.g. silent CREATE TABLE skips)." + depends_on: + - task_key: sync_to_lakebase + notebook_task: + notebook_path: ./10_smoke_check_lakebase.py + base_parameters: + lakebase_dns: ${var.lakebase_dns} + lakebase_database: ${var.lakebase_database} + lakebase_instance: ${var.lakebase_instance} + lakebase_endpoint_path: ${var.lakebase_endpoint_path} + environment_key: default + environments: - environment_key: default spec: