Support bundle deployment bind for DMS-recorded deployments

acceptance/bundle/deployment/bind/test.toml

@@ -1,2 +1,4 @@
-# Bind operations are not yet supported by the Deployment Metadata Service (DMS)
+# These tests cover the file-based bind, which stages a change for the next deploy. Under the
+# deployment metadata service (DMS) bind is a distinct operation - it plans and applies the change
+# immediately - so it cannot share these goldens and is covered by bundle/dms/bind instead.
 EnvMatrix.DMS = [""]

acceptance/bundle/dms/bind/databricks.yml.tmpl

@@ -0,0 +1,18 @@
+bundle:
+  name: dms-bind-$UNIQUE_NAME
+experimental:
+  deployment_history: true
+
+resources:
+  schemas:
+    sc:
+      name: dms_bind_$UNIQUE_NAME
+      catalog_name: main
+  jobs:
+    foo:
+      name: foo
+      permissions:
+        - group_name: users
+          level: CAN_MANAGE
+    bar:
+      name: bar

acceptance/bundle/dms/bind/output.txt

@@ -0,0 +1,110 @@
+
+>>> [CLI] schemas create dms_bind_[UNIQUE_NAME] main
+{
+  "browse_only": false,
+  "catalog_name": "main",
+  "catalog_type": "MANAGED_CATALOG",
+  "created_at": [UNIX_TIME_MILLIS][0],
+  "created_by": "[USERNAME]",
+  "effective_predictive_optimization_flag": {
+    "inherited_from_name": "[METASTORE_NAME]",
+    "inherited_from_type": "METASTORE",
+    "value": "ENABLE"
+  },
+  "enable_predictive_optimization": "INHERIT",
+  "full_name": "main.dms_bind_[UNIQUE_NAME]",
+  "metastore_id": "[UUID]",
+  "name": "dms_bind_[UNIQUE_NAME]",
+  "owner": "[USERNAME]",
+  "schema_id": "[UUID]",
+  "updated_at": [UNIX_TIME_MILLIS][0],
+  "updated_by": "[USERNAME]"
+}
+
+=== Bind sc, the first bind: it creates the deployment. The config already matches, so this is a plain bind that records the resource without touching it
+>>> [CLI] bundle deployment bind sc main.dms_bind_[UNIQUE_NAME] --auto-approve
+Successfully bound schema with an id 'main.dms_bind_[UNIQUE_NAME]'
+Run 'bundle deploy' to deploy changes to your workspace
+
+=== Version 1 records only sc, with the plain bind action type; the jobs are left untouched
+>>> print_requests.py --dms --method POST //versions --nostamp
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "operations": [
+      {
+        "resource_key": "schemas.sc",
+        "action_type": "OPERATION_ACTION_TYPE_BIND"
+      }
+    ]
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+=== Bind foo into the existing deployment: its config differs, so this is a bind-and-update that applies the config now
+>>> [CLI] bundle deployment bind foo [FOO_ID] --auto-approve
+Successfully bound job with an id '[FOO_ID]'
+Run 'bundle deploy' to deploy changes to your workspace
+
+=== Version 2 records only foo and its permissions (bar is untouched) and chains to version 1
+>>> print_requests.py --dms --method POST //versions --nostamp
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "previous_version_id": "1",
+    "operations": [
+      {
+        "resource_key": "jobs.foo",
+        "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE"
+      },
+      {
+        "resource_key": "jobs.foo.permissions",
+        "action_type": "OPERATION_ACTION_TYPE_CREATE"
+      }
+    ]
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+=== sc and foo are now managed; bar is still an unmanaged create
+>>> [CLI] bundle plan
+create jobs.bar
+
+Plan: 1 to add, 0 to change, 0 to delete, 3 unchanged
+
+=== Test cleanup: the bound resources are now bundle-managed, so destroy removes them
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+  delete resources.schemas.sc
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.sc
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-bind-[UNIQUE_NAME]/default
+
+Destroy: 2 deleted

acceptance/bundle/dms/bind/script

@@ -0,0 +1,29 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+# sc's config matches this existing schema exactly, so binding it changes nothing (a plain bind).
+trace $CLI schemas create "dms_bind_$UNIQUE_NAME" main
+# foo's config ("foo") differs from this existing job, so binding it applies an update.
+foo_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
+add_repl "$foo_id" FOO_ID
+
+cleanup() {
+    title "Test cleanup: the bound resources are now bundle-managed, so destroy removes them"
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "Bind sc, the first bind: it creates the deployment. The config already matches, so this is a plain bind that records the resource without touching it"
+trace $CLI bundle deployment bind sc "main.dms_bind_$UNIQUE_NAME" --auto-approve
+
+title "Version 1 records only sc, with the plain bind action type; the jobs are left untouched"
+trace print_requests.py --dms --method POST //versions --nostamp
+
+title "Bind foo into the existing deployment: its config differs, so this is a bind-and-update that applies the config now"
+trace $CLI bundle deployment bind foo "$foo_id" --auto-approve
+
+title "Version 2 records only foo and its permissions (bar is untouched) and chains to version 1"
+trace print_requests.py --dms --method POST //versions --nostamp
+
+title "sc and foo are now managed; bar is still an unmanaged create"
+trace $CLI bundle plan

acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl → acceptance/bundle/dms/record-op-fault/databricks.yml.tmpl

@@ -1,5 +1,5 @@
 bundle:
-  name: dms-bind-unbind-$UNIQUE_NAME
+  name: dms-record-op-fault-$UNIQUE_NAME
 experimental:
   deployment_history: true
 

acceptance/bundle/dms/record-op-fault/output.txt

@@ -0,0 +1,39 @@
+
+=== The service rejects the operation upload, so recording the bind fails
+>>> fault.py PATCH /api/2.0/bundle/* 500 0 1
+
+>>> musterr [CLI] bundle deployment bind foo [JOB_ID] --auto-approve
+Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INJECTED)
+
+Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus
+HTTP Status: 500 Internal Server Error
+API error_code: INJECTED
+API message: Fault injected by test.
+
+
+=== The version is still completed - with failure - rather than left holding its lease
+>>> print_requests.py --dms --method POST //versions
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "operations": [
+      {
+        "resource_key": "jobs.foo",
+        "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE"
+      }
+    ]
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_FAILURE"
+  }
+}

acceptance/bundle/dms/record-op-fault/script

@@ -0,0 +1,11 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+job_id=$($CLI jobs create --json '{"name": "target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
+add_repl "$job_id" JOB_ID
+
+title "The service rejects the operation upload, so recording the bind fails"
+trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1
+trace musterr $CLI bundle deployment bind foo "$job_id" --auto-approve
+
+title "The version is still completed - with failure - rather than left holding its lease"
+trace print_requests.py --dms --method POST //versions

acceptance/bundle/dms/record-op-fault/test.toml

@@ -0,0 +1,3 @@
+# Local only: the case needs an injected failure (DMS must reject the operation upload once), and
+# fault.py registers its rule on the fake, which a cloud run does not have.
+Cloud = false

bundle/deployplan/action.go

@@ -31,25 +31,32 @@ type ActionType string
 // If case of several options, action with highest severity wins.
 // Note, Create/Delete are handled explicitly and never compared.
 const (
-	Undefined    ActionType = ""
-	Skip         ActionType = "skip"
-	Resize       ActionType = "resize"
-	Update       ActionType = "update"
-	UpdateWithID ActionType = "update_id"
-	Create       ActionType = "create"
-	Recreate     ActionType = "recreate"
-	Delete       ActionType = "delete"
+	Undefined ActionType = ""
+	Skip      ActionType = "skip"
+	Resize    ActionType = "resize"
+	Update    ActionType = "update"
+	// Bind adopts an existing workspace resource whose config already matches it: record it in
+	// state without touching the resource. BindAndUpdate adopts it and applies the config in one
+	// step. Both are set only for the resource named by `bundle deployment bind`.
+	Bind          ActionType = "bind"
+	BindAndUpdate ActionType = "bind_and_update"
+	UpdateWithID  ActionType = "update_id"
+	Create        ActionType = "create"
+	Recreate      ActionType = "recreate"
+	Delete        ActionType = "delete"
 )
 
 var actionOrder = map[ActionType]int{
-	Undefined:    0,
-	Skip:         1,
-	Resize:       2,
-	Update:       3,
-	UpdateWithID: 4,
-	Create:       5,
-	Recreate:     6,
-	Delete:       7,
+	Undefined:     0,
+	Skip:          1,
+	Bind:          2,
+	Resize:        3,
+	Update:        4,
+	BindAndUpdate: 5,
+	UpdateWithID:  6,
+	Create:        7,
+	Recreate:      8,
+	Delete:        9,
 }
 
 func (a ActionType) KeepsID() bool {

bundle/deployplan/plan.go

@@ -54,15 +54,16 @@ func (p *Plan) CountActions() ActionCounts {
 		switch entry.Action {
 		case Create:
 			c.Create++
-		case Update, UpdateWithID, Resize:
+		case Update, UpdateWithID, Resize, BindAndUpdate:
 			c.Change++
 		case Delete:
 			c.Delete++
 		case Recreate:
 			// A recreate counts as both a delete and a create.
 			c.Delete++
 			c.Create++
-		case Skip, Undefined:
+		case Skip, Undefined, Bind:
+			// Bind adopts an existing resource without changing it.
 			c.Unchanged++
 		}
 	}
@@ -116,6 +117,8 @@ func LoadPlanFromFile(path string) (*Plan, error) {
 }
 
 type PlanEntry struct {
+	// ID carries the workspace id for a Bind/BindAndUpdate action, whose resource is not yet in
+	// state; every other action reads the id from state instead, so it is empty for them.
 	ID        string           `json:"id,omitempty"`
 	DependsOn []DependsOnEntry `json:"depends_on,omitempty"`
 	Action    ActionType       `json:"action,omitempty"`

bundle/direct/apply.go

@@ -35,8 +35,18 @@ func (d *DeploymentUnit) Destroy(ctx context.Context, db *dstate.DeploymentState
 func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, newState any, actionType deployplan.ActionType, planEntry *deployplan.PlanEntry) error {
 	ctx = log.WithPrefix(ctx, "deploying "+d.ResourceKey)
 	ctx = d.withResourceKey(ctx)
-	if actionType == deployplan.Create {
+
+	// Bind adopts an existing resource, so its id comes from the plan entry rather than state.
+	switch actionType {
+	case deployplan.Create:
 		return d.Create(ctx, db, newState)
+	case deployplan.Bind:
+		return d.Bind(ctx, db, planEntry.ID, newState, planEntry)
+	case deployplan.BindAndUpdate:
+		// Adopt and apply the config in one step; the update is the same as any other.
+		return d.Update(ctx, db, planEntry.ID, newState, planEntry)
+	default:
+		// The remaining actions act on a resource already in state; handled below.
 	}
 
 	oldID := db.GetResourceID(d.ResourceKey)
@@ -58,6 +68,15 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState,
 	}
 }
 
+// Bind adopts an existing workspace resource: it records the id and config as state without any
+// API write, since the config already matches the resource. The remote etag is copied in so
+// etag-based drift detection (dashboards, genie_spaces) still works on the next plan.
+func (d *DeploymentUnit) Bind(ctx context.Context, db *dstate.DeploymentState, id string, newState any, planEntry *deployplan.PlanEntry) error {
+	copyRemoteEtag(d.ResourceKey, planEntry.RemoteState, newState)
+	log.Infof(ctx, "Bound %s id=%#v", d.ResourceKey, id)
+	return d.saveState(ctx, db, id, newState, d.DependsOn)
+}
+
 // Create creates the resource and records its state.
 func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error {
 	var newID string