From 9d45d976a085a5b655387a2b4aa30616995ae3e1 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 16:20:28 +0200 Subject: [PATCH 1/6] direct: ignore UC-managed schema properties as backend defaults UC auto-populates system-managed property keys (e.g. `unity.catalog.managed.delta.defaults.delta.enableRowTracking`) on schema creation. Without a backend_defaults rule, the planner sees the remote map as drift, emits Update, and DoUpdate sends an empty payload which UC rejects with "UpdateSchema Nothing to update". The rule only applies when both saved and new are nil, so user-set properties still drive real drift detection. Also mirror the UC behavior in the fake testserver so the no-drift invariant is exercised locally; added acceptance/.../schemas/drift/ managed_properties covering the reproducer. Co-authored-by: Isaac --- .../drift/managed_properties/databricks.yml | 8 +++++++ .../drift/managed_properties/out.test.toml | 3 +++ .../drift/managed_properties/output.txt | 24 +++++++++++++++++++ .../schemas/drift/managed_properties/script | 11 +++++++++ .../drift/managed_properties/test.toml | 5 ++++ .../resources/schemas/recreate/output.txt | 3 +++ bundle/direct/dresources/resources.yml | 6 +++++ libs/testserver/schemas.go | 7 ++++++ 8 files changed, 67 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/output.txt create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/script create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/test.toml diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml new file mode 100644 index 00000000000..cd05d222c48 --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test-bundle + +resources: + schemas: + schema1: + name: myschema + catalog_name: main diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt new file mode 100644 index 00000000000..869cd97c29d --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt @@ -0,0 +1,24 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op despite UC auto-populating managed properties +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Redeploy is a no-op (no UpdateSchema call) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity +json.method = "POST"; +json.path = "/api/2.1/unity-catalog/schemas"; +json.body.catalog_name = "main"; +json.body.name = "myschema"; diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/script b/acceptance/bundle/resources/schemas/drift/managed_properties/script new file mode 100644 index 00000000000..c3ef4ac39e4 --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/script @@ -0,0 +1,11 @@ +echo "*" > .gitignore + +title "Initial deployment" +trace $CLI bundle deploy + +title "Plan is a no-op despite UC auto-populating managed properties" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "Redeploy is a no-op (no UpdateSchema call)" +trace $CLI bundle deploy +trace print_requests.py //unity | gron.py | contains.py '!json.method = "PATCH"' diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/test.toml new file mode 100644 index 00000000000..5016e85b395 --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/test.toml @@ -0,0 +1,5 @@ +RecordRequests = true + +# Terraform issues a spurious PATCH for enable_predictive_optimization on every +# deploy, which is outside the scope of backend-default handling in resources.yml. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt index 7c173eb11f0..bb33023f9d2 100644 --- a/acceptance/bundle/resources/schemas/recreate/output.txt +++ b/acceptance/bundle/resources/schemas/recreate/output.txt @@ -83,6 +83,9 @@ Error: Resource catalog.SchemaInfo not found: main.myschema "metastore_id": "[UUID]", "name": "myschema", "owner": "[USERNAME]", + "properties": { + "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true" + }, "schema_id": "[UUID]", "updated_at": [UNIX_TIME_MILLIS][0], "updated_by": "[USERNAME]" diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 569fca9ee82..3375908f3ba 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -323,6 +323,12 @@ resources: reason: immutable - field: storage_root reason: immutable + backend_defaults: + # UC auto-populates system-managed keys like + # `unity.catalog.managed.delta.defaults.delta.enableRowTracking` after create. + # Without this, every subsequent plan produces an Update whose payload is empty, + # and UC rejects it with "UpdateSchema Nothing to update". + - field: properties external_locations: recreate_on_changes: diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 1d4dc79e7ac..92c01bf7fc2 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -39,6 +39,13 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { schema.MetastoreId = TestMetastore.MetastoreId schema.Owner = s.CurrentUser().UserName schema.SchemaId = nextUUID() + if schema.Properties == nil { + // Mirror UC behavior: managed system defaults are populated when the user + // doesn't specify any properties. Required to cover backend-default drift. + schema.Properties = map[string]string{ + "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", + } + } s.Schemas[schema.FullName] = schema return Response{ From 1f336d8d738bcae71bff5687d0ccb68666e5623e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 18:18:15 +0200 Subject: [PATCH 2/6] Narrow properties --- bundle/direct/bundle_plan_test.go | 61 ++++++++++++++++++++++++++ bundle/direct/dresources/resources.yml | 6 +-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index ccfb7cb517f..1579217ae52 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -3,8 +3,12 @@ package direct import ( "testing" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDynPathToStructPath(t *testing.T) { @@ -35,3 +39,60 @@ func TestDynPathToStructPath(t *testing.T) { assert.Equal(t, tc.expected, node.String()) } } + +func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { + cfg := dresources.GetResourceConfig("schemas") + require.NotNil(t, cfg) + + tests := []struct { + name string + path string + remote any + expected bool + }{ + { + name: "managed delta row tracking property", + path: "properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking']", + remote: "true", + expected: true, + }, + { + name: "managed iceberg catalog property", + path: "properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged']", + remote: "true", + expected: true, + }, + { + name: "unmanaged remote-only property is not skipped", + path: "properties['custom.remote_only']", + remote: "true", + expected: false, + }, + { + name: "parent properties map is not skipped", + path: "properties", + remote: map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true"}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path, err := structpath.ParsePath(tt.path) + require.NoError(t, err) + + reason, ok := shouldSkipBackendDefault(cfg, path, &deployplan.ChangeDesc{ + Old: nil, + New: nil, + Remote: tt.remote, + }) + + assert.Equal(t, tt.expected, ok) + if tt.expected { + assert.Equal(t, deployplan.ReasonBackendDefault, reason) + } else { + assert.Empty(t, reason) + } + }) + } +} diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 3375908f3ba..1d07bbf4ae7 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -324,11 +324,11 @@ resources: - field: storage_root reason: immutable backend_defaults: - # UC auto-populates system-managed keys like - # `unity.catalog.managed.delta.defaults.delta.enableRowTracking` after create. + # UC auto-populates these system-managed keys after create. # Without this, every subsequent plan produces an Update whose payload is empty, # and UC rejects it with "UpdateSchema Nothing to update". - - field: properties + - field: properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking'] + - field: properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged'] external_locations: recreate_on_changes: From acc3518cbe638e75d9b701238957a10d12585bcd Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 18:26:37 +0200 Subject: [PATCH 3/6] direct: handle schema backend-default map drift --- bundle/direct/bundle_plan.go | 51 ++++++++++++++++++++++++++----- bundle/direct/bundle_plan_test.go | 28 ++++++++++++++++- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f6bcea316cd..d53c185ed83 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -452,19 +452,56 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str return "", false } for _, rule := range cfg.BackendDefaults { - if !path.HasPatternPrefix(rule.Field) { - continue - } - if len(rule.Values) == 0 { - return deployplan.ReasonBackendDefault, true - } - if matchesAllowedValue(ch.Remote, rule.Values) { + if matchesBackendDefaultRule(path, ch.Remote, rule) { return deployplan.ReasonBackendDefault, true } } + if matchesBackendDefaultMap(cfg, path, ch.Remote) { + return deployplan.ReasonBackendDefault, true + } return "", false } +func matchesBackendDefaultRule(path *structpath.PathNode, remote any, rule dresources.BackendDefaultRule) bool { + if !path.HasPatternPrefix(rule.Field) { + return false + } + if len(rule.Values) == 0 { + return true + } + return matchesAllowedValue(remote, rule.Values) +} + +// matchesBackendDefaultMap handles the nil-vs-map case from structdiff, where a +// remote-only map change is emitted at the parent path rather than per key. +// We only skip the parent map if every remote entry matches a configured +// backend-default child rule; any unmanaged key must still surface as drift. +func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { + rv := reflect.ValueOf(remote) + if !rv.IsValid() || rv.Kind() != reflect.Map || rv.IsNil() || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 { + return false + } + + iter := rv.MapRange() + for iter.Next() { + childPath := structpath.NewBracketString(path, iter.Key().String()) + childRemote := iter.Value().Interface() + + matched := false + for _, rule := range cfg.BackendDefaults { + if matchesBackendDefaultRule(childPath, childRemote, rule) { + matched = true + break + } + } + if !matched { + return false + } + } + + return true +} + // matchesAllowedValue checks if the remote value matches one of the allowed JSON values. // Each json.RawMessage is unmarshaled into the same type as remote for comparison. func matchesAllowedValue(remote any, values []json.RawMessage) bool { diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 1579217ae52..0b3191aca30 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -69,9 +69,15 @@ func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { expected: false, }, { - name: "parent properties map is not skipped", + name: "managed-only parent properties map is skipped", path: "properties", remote: map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true"}, + expected: true, + }, + { + name: "mixed parent properties map is not skipped", + path: "properties", + remote: map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", "custom.remote_only": "true"}, expected: false, }, } @@ -96,3 +102,23 @@ func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { }) } } + +// Map drift handling synthesizes child paths to match against rules. structdiff +// always emits map keys in bracket notation, so synthetic child paths must too; +// otherwise rules wouldn't match for identifier-like keys. +func TestShouldSkipBackendDefault_MapDriftUsesBracketKeys(t *testing.T) { + field, err := structpath.ParsePattern("properties['simple']") + require.NoError(t, err) + cfg := &dresources.ResourceLifecycleConfig{ + BackendDefaults: []dresources.BackendDefaultRule{{Field: field}}, + } + + path, err := structpath.ParsePath("properties") + require.NoError(t, err) + + reason, ok := shouldSkipBackendDefault(cfg, path, &deployplan.ChangeDesc{ + Remote: map[string]string{"simple": "v"}, + }) + assert.True(t, ok) + assert.Equal(t, deployplan.ReasonBackendDefault, reason) +} From 483d288eefc0d101cc9fd3206bc8c1d8b53c8129 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 9 Jun 2026 17:25:12 +0200 Subject: [PATCH 4/6] testserver: scope schema managed-defaults to the drift test The fake populated UC-managed properties on every property-less schema. The direct engine ignores them as backend defaults, but terraform treats them as drift, which broke unrelated schema acceptance tests across both engines. Gate the injection on a dedicated schema name so only the backend-default drift test opts in, and drop recreate's now-incidental property line. Co-authored-by: Isaac --- .../schemas/drift/managed_properties/databricks.yml | 2 +- .../resources/schemas/drift/managed_properties/output.txt | 2 +- acceptance/bundle/resources/schemas/recreate/output.txt | 3 --- libs/testserver/schemas.go | 8 +++++++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml index cd05d222c48..5ab0d85ca59 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml @@ -4,5 +4,5 @@ bundle: resources: schemas: schema1: - name: myschema + name: schema_managed_defaults catalog_name: main diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt index 869cd97c29d..f2ecb3506f0 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt @@ -21,4 +21,4 @@ Deployment complete! json.method = "POST"; json.path = "/api/2.1/unity-catalog/schemas"; json.body.catalog_name = "main"; -json.body.name = "myschema"; +json.body.name = "schema_managed_defaults"; diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt index a86a556a22b..39cafa5b4c3 100644 --- a/acceptance/bundle/resources/schemas/recreate/output.txt +++ b/acceptance/bundle/resources/schemas/recreate/output.txt @@ -83,9 +83,6 @@ Error: Resource catalog.SchemaInfo not found: main.myschema "metastore_id": "[UUID]", "name": "myschema", "owner": "[USERNAME]", - "properties": { - "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true" - }, "schema_id": "[UUID]", "updated_at": [UNIX_TIME_MILLIS][0], "updated_by": "[USERNAME]" diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 92c01bf7fc2..883133b8d38 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -11,6 +11,12 @@ import ( const testMetastoreName = "deco-uc-prod-isolated-aws-us-east-1" +// schemaNameManagedDefaults is the schema name the backend-default drift test uses +// to opt into UC's managed-property simulation. Scoping the injection to this name +// keeps unrelated schema tests free of the property, which terraform would otherwise +// report as drift on redeploy. +const schemaNameManagedDefaults = "schema_managed_defaults" + func (s *FakeWorkspace) SchemasCreate(req Request) Response { defer s.LockUnlock()() @@ -39,7 +45,7 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { schema.MetastoreId = TestMetastore.MetastoreId schema.Owner = s.CurrentUser().UserName schema.SchemaId = nextUUID() - if schema.Properties == nil { + if schema.Properties == nil && schema.Name == schemaNameManagedDefaults { // Mirror UC behavior: managed system defaults are populated when the user // doesn't specify any properties. Required to cover backend-default drift. schema.Properties = map[string]string{ From 56952a761cf485469fe14ad83da489cf513ad6b4 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 10 Jun 2026 14:16:24 +0200 Subject: [PATCH 5/6] Factor out asNonEmptyStringMap inside matchesBackendDefaultMap --- bundle/direct/bundle_plan.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 0fa86359f4d..fff3de0719f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -480,8 +480,8 @@ func matchesBackendDefaultRule(path *structpath.PathNode, remote any, rule dreso // We only skip the parent map if every remote entry matches a configured // backend-default child rule; any unmanaged key must still surface as drift. func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { - rv := reflect.ValueOf(remote) - if !rv.IsValid() || rv.Kind() != reflect.Map || rv.IsNil() || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 { + rv, ok := asNonEmptyStringMap(remote) + if !ok { return false } @@ -505,6 +505,16 @@ func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *str return true } +// asNonEmptyStringMap returns remote as a reflected map value when it is a +// non-nil, non-empty map with string keys; ok is false otherwise. +func asNonEmptyStringMap(remote any) (reflect.Value, bool) { + rv := reflect.ValueOf(remote) + if !rv.IsValid() || rv.Kind() != reflect.Map || rv.IsNil() || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 { + return reflect.Value{}, false + } + return rv, true +} + // matchesAllowedValue checks if the remote value matches one of the allowed JSON values. // Each json.RawMessage is unmarshaled into the same type as remote for comparison. func matchesAllowedValue(remote any, values []json.RawMessage) bool { From 66711bf5393d367282209c5dde974532dbbd7dfb Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 10 Jun 2026 14:19:31 +0200 Subject: [PATCH 6/6] Factor out matchesAnyBackendDefault and use in both places --- bundle/direct/bundle_plan.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index fff3de0719f..d389cb931c9 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -454,17 +454,23 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str if cfg == nil || ch.Old != nil || ch.New != nil || ch.Remote == nil { return "", false } - for _, rule := range cfg.BackendDefaults { - if matchesBackendDefaultRule(path, ch.Remote, rule) { - return deployplan.ReasonBackendDefault, true - } - } - if matchesBackendDefaultMap(cfg, path, ch.Remote) { + if matchesAnyBackendDefault(cfg, path, ch.Remote) || matchesBackendDefaultMap(cfg, path, ch.Remote) { return deployplan.ReasonBackendDefault, true } return "", false } +// matchesAnyBackendDefault reports whether the change at path matches any of the +// resource's configured backend-default rules. +func matchesAnyBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { + for _, rule := range cfg.BackendDefaults { + if matchesBackendDefaultRule(path, remote, rule) { + return true + } + } + return false +} + func matchesBackendDefaultRule(path *structpath.PathNode, remote any, rule dresources.BackendDefaultRule) bool { if !path.HasPatternPrefix(rule.Field) { return false @@ -488,16 +494,7 @@ func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *str iter := rv.MapRange() for iter.Next() { childPath := structpath.NewBracketString(path, iter.Key().String()) - childRemote := iter.Value().Interface() - - matched := false - for _, rule := range cfg.BackendDefaults { - if matchesBackendDefaultRule(childPath, childRemote, rule) { - matched = true - break - } - } - if !matched { + if !matchesAnyBackendDefault(cfg, childPath, iter.Value().Interface()) { return false } }