fix(workflows): gateway DDL + billing perm-error + smoke check + tools tables + CI/CD plan

.github/workflows/deploy-sandbox.yml

@@ -0,0 +1,97 @@
+name: Deploy → sandbox
+
+# Triggers a Databricks bundle deploy (workflow + app) into the sandbox
+# workspace, then triggers the discovery workflow once and gates promotion on
+# the smoke_check_lakebase task. See docs/decisions/2026-06-06-cicd-deployment-pattern.md.
+#
+# This is the SANDBOX gate. Stage and prod deployments use sibling workflows
+# with their own secrets and approval rules.
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+env:
+  DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST_SANDBOX }}
+  DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID_SANDBOX }}
+  DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET_SANDBOX }}
+  BUNDLE_TARGET: dev
+
+jobs:
+  deploy-and-smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: control-plane-app/package-lock.json
+
+      - name: Install Databricks CLI
+        uses: databricks/setup-cli@main
+
+      - name: Build frontend
+        working-directory: control-plane-app
+        run: |
+          npm ci
+          npm run build
+
+      - name: Bundle validate
+        working-directory: workflows
+        run: databricks bundle validate --target "$BUNDLE_TARGET"
+
+      - name: Bundle deploy
+        working-directory: workflows
+        run: databricks bundle deploy --target "$BUNDLE_TARGET"
+
+      - name: Deploy app
+        working-directory: control-plane-app
+        run: bash deploy.sh
+
+      - name: Trigger discovery workflow
+        id: trigger
+        run: |
+          # Find the deployed job ID from bundle state.
+          JOB_ID=$(databricks bundle summary --target "$BUNDLE_TARGET" --output json \
+            --working-dir workflows \
+            | python3 -c "import sys,json; d=json.load(sys.stdin); jobs=d.get('resources',{}).get('jobs',{}); print(next(iter(jobs.values())).get('id',''))")
+          if [ -z "$JOB_ID" ]; then
+            echo "Could not resolve job ID from bundle summary"; exit 1
+          fi
+          echo "Triggering job $JOB_ID..."
+          RUN_ID=$(databricks jobs run-now "$JOB_ID" --output json | python3 -c "import sys,json; print(json.load(sys.stdin).get('run_id',''))")
+          echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"
+          echo "Triggered run $RUN_ID"
+
+      - name: Wait for run + gate on smoke check
+        run: |
+          RUN_ID="${{ steps.trigger.outputs.run_id }}"
+          # Poll until terminal. The smoke task runs last in the DAG; if it fails,
+          # the whole run reports result_state=FAILED.
+          while true; do
+            STATE=$(databricks jobs get-run "$RUN_ID" --output json \
+              | python3 -c "import sys,json; d=json.load(sys.stdin); s=d.get('state',{}); print(s.get('life_cycle_state','')+'/'+(s.get('result_state') or '-'))")
+            echo "$(date -u +%FT%TZ) state=$STATE"
+            case "$STATE" in
+              TERMINATED/SUCCESS) echo "✅ Discovery + smoke passed."; exit 0 ;;
+              TERMINATED/*|INTERNAL_ERROR/*|SKIPPED/*)
+                echo "❌ Discovery run terminal with non-success: $STATE"
+                # Surface per-task results to the CI log so the smoke FAIL message lands
+                databricks jobs get-run "$RUN_ID" --output json \
+                  | python3 -c "
+                  import sys, json
+                  d = json.load(sys.stdin)
+                  for t in d.get('tasks', []):
+                      s = t.get('state', {})
+                      print(f\"  {t.get('task_key'):35s} {s.get('life_cycle_state'):12s} {s.get('result_state','-')}\")"
+                exit 1 ;;
+              *) sleep 30 ;;
+            esac
+          done

.gitignore

@@ -35,3 +35,5 @@ Thumbs.db
 
 # Local brainstorm / parking-lot notes (not part of the project)
 ideas/
+.claude/
+CLAUDE.md

control-plane-app/backend/services/tools_service.py

@@ -53,8 +53,8 @@ def ensure_tools_tables():
     for stmt in ddl_statements:
         try:
             execute_update(stmt)
-        except Exception as exc:
-            logger.warning("Tools DDL warning: %s", exc)
+        except Exception:
+            logger.exception("Tools DDL failed (statement=%s)", stmt[:80])
     logger.info("Tools tables ensured")
 
 
@@ -451,11 +451,17 @@ def refresh_tools():
         _refresh_in_progress = True
         logger.info("Starting tools discovery …")
 
+        # Self-heal: the table is normally created by workflows/02_sync_to_lakebase
+        # Phase 7, but calling this defensively here means the Tools page works
+        # even if the workflow hasn't run yet (fresh deploy) or if the app SP
+        # somehow lost its read-only access to that table at boot.
+        ensure_tools_tables()
+
         # Clear old MCP entries (they may be stale serving-endpoint records)
         try:
             execute_update("DELETE FROM tool_registry WHERE type = 'mcp_server'")
-        except Exception:
-            pass
+        except Exception as exc:
+            logger.warning("DELETE FROM tool_registry failed: %s", exc)
 
         # 1) Managed MCP — UC connections with is_mcp_connection
         mcp_conns = _discover_mcp_connections()
@@ -481,8 +487,12 @@ def refresh_tools():
         logger.info("   → UC functions: %s", len(funcs))
 
         logger.info("Tools discovery complete")
-    except Exception as exc:
-        logger.warning("Tools refresh failed: %s", exc)
+    except Exception:
+        # Log with traceback. Returning silently here is what hid the
+        # "tool_registry does not exist" failure observed during onboarding —
+        # /api/v1/tools/sync answered 200 while the underlying refresh
+        # blew up on the missing table.
+        logger.exception("Tools refresh failed")
     finally:
         _refresh_in_progress = False
         _refresh_lock.release()

control-plane-app/deploy.sh

@@ -134,18 +134,10 @@ echo "Deployment triggered. Monitor with:"
 echo "  databricks apps get $APP_NAME $PROFILE_FLAG"
 
 # ── Grant the app SP access to Lakebase ───────────────────────
-# Idempotent — safe to re-run. Needs psycopg2 and databricks-sdk locally.
+# Some workspaces block public Lakebase Postgres connectivity from the laptop
+# ("Public access is not allowed for workspace ..."). When this happens, run
+# the equivalent grant inside the workspace as a one-shot job instead — see
+# run_grant_sp_lakebase_job.sh in this directory.
 echo ""
-echo "Granting app SP access to Lakebase ..."
-PROFILE_ENV=""
-if [[ -n "$PROFILE_FLAG" ]]; then
-  PROFILE_ENV="DATABRICKS_CONFIG_PROFILE=${PROFILE_FLAG#--profile }"
-fi
-env $PROFILE_ENV \
-  APP_NAME="$APP_NAME" \
-  LAKEBASE_DNS="$LAKEBASE_DNS" \
-  LAKEBASE_DATABASE="$LAKEBASE_DATABASE" \
-  LAKEBASE_ENDPOINT_PATH="${LAKEBASE_ENDPOINT_PATH:-}" \
-  LAKEBASE_INSTANCE="${LAKEBASE_INSTANCE:-}" \
-  python3 grant_sp_lakebase.py \
-  || echo "  ⚠ grant_sp_lakebase.py failed — you may need to run it manually. See docs/installation.md."
+echo "Skipping local grant_sp_lakebase.py — run via in-workspace job:"
+echo "  bash run_grant_sp_lakebase_job.sh ${PROFILE_FLAG:-}"

control-plane-app/grant_sp_lakebase_notebook.py

@@ -0,0 +1,42 @@
+# Databricks notebook source
+# MAGIC %md
+# MAGIC # grant_sp_lakebase wrapper
+# MAGIC In-workspace runner for `grant_sp_lakebase.py` — reads widget params, sets
+# MAGIC them in the environment, then exec's the original script.
+
+# COMMAND ----------
+
+# MAGIC %pip install --upgrade "databricks-sdk>=0.40.0" psycopg2-binary requests
+# MAGIC dbutils.library.restartPython()
+
+# COMMAND ----------
+
+import os, runpy
+
+dbutils.widgets.text("app_name", "", "App name")
+dbutils.widgets.text("lakebase_dns", "", "Lakebase DNS")
+dbutils.widgets.text("lakebase_database", "", "Lakebase database")
+dbutils.widgets.text("lakebase_instance", "", "Lakebase instance (Provisioned)")
+dbutils.widgets.text("lakebase_endpoint_path", "", "Lakebase endpoint path (Autoscaling)")
+dbutils.widgets.text("script_path", "", "Path to grant_sp_lakebase.py in /Workspace")
+
+os.environ["APP_NAME"] = dbutils.widgets.get("app_name")
+os.environ["LAKEBASE_DNS"] = dbutils.widgets.get("lakebase_dns")
+os.environ["LAKEBASE_DATABASE"] = dbutils.widgets.get("lakebase_database")
+os.environ["LAKEBASE_INSTANCE"] = dbutils.widgets.get("lakebase_instance")
+os.environ["LAKEBASE_ENDPOINT_PATH"] = dbutils.widgets.get("lakebase_endpoint_path")
+
+script_path = dbutils.widgets.get("script_path")
+print(f"Running grant_sp_lakebase.py from {script_path}")
+print(f"  APP_NAME={os.environ['APP_NAME']}")
+print(f"  LAKEBASE_DNS={os.environ['LAKEBASE_DNS']}")
+print(f"  LAKEBASE_INSTANCE={os.environ['LAKEBASE_INSTANCE']}")
+
+try:
+    runpy.run_path(script_path, run_name="__main__")
+    print("grant_sp_lakebase: completed without raising")
+except SystemExit as e:
+    code = e.code if e.code is not None else 0
+    print(f"grant_sp_lakebase: exited with code {code}")
+    if code != 0:
+        raise RuntimeError(f"grant_sp_lakebase.py exited with non-zero code {code}")

control-plane-app/run_grant_sp_lakebase_job.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# Run grant_sp_lakebase.py as a one-shot Databricks job inside the workspace.
+#
+# Use this when the local machine cannot reach Lakebase Postgres directly
+# (e.g. when the workspace blocks public PG access). The job runs on
+# serverless compute as the deploying user, who is the Lakebase admin.
+#
+# Reads .env from the current directory for APP_NAME / LAKEBASE_* settings.
+# Pass --profile <name> to use a non-default Databricks CLI profile.
+
+set -euo pipefail
+
+PROFILE_FLAG=""
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --profile) PROFILE_FLAG="--profile $2"; shift 2 ;;
+    *) echo "Unknown option: $1"; exit 1 ;;
+  esac
+done
+
+if [ ! -f .env ]; then
+  echo "Error: .env not found in $(pwd). Run this from control-plane-app/."
+  exit 1
+fi
+
+# Load .env
+set -a
+while IFS='=' read -r key value; do
+  [[ -z "$key" || "$key" =~ ^# ]] && continue
+  export "$key=$value"
+done < .env
+set +a
+
+: "${APP_NAME:?Set APP_NAME in .env}"
+: "${LAKEBASE_DNS:?Set LAKEBASE_DNS in .env}"
+: "${LAKEBASE_DATABASE:?Set LAKEBASE_DATABASE in .env}"
+LAKEBASE_INSTANCE="${LAKEBASE_INSTANCE:-}"
+LAKEBASE_ENDPOINT_PATH="${LAKEBASE_ENDPOINT_PATH:-}"
+
+DB="databricks"
+WORKSPACE_USER=$($DB auth describe $PROFILE_FLAG 2>/dev/null | grep -i "user" | head -1 | awk '{print $NF}')
+WORKSPACE_DIR="/Workspace/Users/${WORKSPACE_USER}/ai-control-plane/control-plane-app"
+SCRIPT_PATH="${WORKSPACE_DIR}/grant_sp_lakebase.py"
+NB_PATH="${WORKSPACE_DIR}/grant_sp_lakebase_notebook"
+
+echo "Uploading grant_sp_lakebase.py ..."
+$DB workspace import "$SCRIPT_PATH" \
+  --file grant_sp_lakebase.py \
+  --format AUTO --overwrite $PROFILE_FLAG
+
+echo "Uploading grant_sp_lakebase_notebook.py ..."
+$DB workspace import "$NB_PATH" \
+  --file grant_sp_lakebase_notebook.py \
+  --format SOURCE --language PYTHON --overwrite $PROFILE_FLAG
+
+JOB_JSON=$(cat <<EOF
+{
+  "run_name": "ai-control-plane: grant_sp_lakebase",
+  "tasks": [
+    {
+      "task_key": "grant_sp_lakebase",
+      "notebook_task": {
+        "notebook_path": "${NB_PATH}",
+        "base_parameters": {
+          "app_name": "${APP_NAME}",
+          "lakebase_dns": "${LAKEBASE_DNS}",
+          "lakebase_database": "${LAKEBASE_DATABASE}",
+          "lakebase_instance": "${LAKEBASE_INSTANCE}",
+          "lakebase_endpoint_path": "${LAKEBASE_ENDPOINT_PATH}",
+          "script_path": "${SCRIPT_PATH}"
+        }
+      },
+      "environment_key": "default"
+    }
+  ],
+  "environments": [
+    {
+      "environment_key": "default",
+      "spec": {
+        "client": "1",
+        "dependencies": ["databricks-sdk>=0.40.0", "psycopg2-binary", "requests"]
+      }
+    }
+  ]
+}
+EOF
+)
+
+echo "Submitting one-shot job ..."
+RESULT=$($DB jobs submit --json "$JOB_JSON" $PROFILE_FLAG -o json)
+echo "$RESULT" | python3 -c "import sys,json
+d=json.load(sys.stdin)
+print('  run_id:', d.get('run_id'))
+print('  result:', d.get('state',{}).get('result_state'))
+print('  life_cycle:', d.get('state',{}).get('life_cycle_state'))
+print('  message:', d.get('state',{}).get('state_message',''))"
+
+RUN_ID=$(echo "$RESULT" | python3 -c "import sys,json; print(json.load(sys.stdin).get('run_id',''))")
+
+echo ""
+echo "Task output:"
+TASK_RUN_ID=$($DB jobs get-run "$RUN_ID" $PROFILE_FLAG -o json 2>&1 | python3 -c "import sys,json
+d=json.load(sys.stdin); ts=d.get('tasks',[]); print(ts[0].get('run_id','') if ts else '')")
+$DB jobs get-run-output "$TASK_RUN_ID" $PROFILE_FLAG -o json 2>&1 | python3 -c "
+import sys,json
+try:
+  d=json.load(sys.stdin)
+  if d.get('error'): print('--- error ---'); print(d['error'])
+  if d.get('error_trace'):
+    import re
+    cleaned = re.sub(r'\\x1b\\[[0-9;]*m','',d['error_trace'])
+    print('--- error_trace ---'); print(cleaned[-1500:])
+  if d.get('logs'): print('--- logs ---'); print(d['logs'][-1500:])
+  if d.get('notebook_output',{}).get('result'): print('--- notebook result ---'); print(d['notebook_output']['result'])
+except Exception as e: print('parse err:', e)
+" || true
+
+echo ""
+echo "Run finished. Inspect with:"
+echo "  databricks jobs get-run $RUN_ID $PROFILE_FLAG"