From c5ab8ccee0c021d340e229cc4ac5aa6cb0f2b9b7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 9 Jun 2026 16:10:38 +0200 Subject: [PATCH 1/7] add uppercase schema --- .../configs/schema_uppercase_name.yml.tmpl | 17 +++++++++++++++++ .../bundle/invariant/continue_293/out.test.toml | 1 + .../bundle/invariant/migrate/out.test.toml | 1 + .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 1 + 5 files changed, 21 insertions(+) create mode 100644 acceptance/bundle/invariant/configs/schema_uppercase_name.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/schema_uppercase_name.yml.tmpl b/acceptance/bundle/invariant/configs/schema_uppercase_name.yml.tmpl new file mode 100644 index 00000000000..46e59e26bc9 --- /dev/null +++ b/acceptance/bundle/invariant/configs/schema_uppercase_name.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +# Reproduce: UC API normalizes identifier names to lowercase, causing +# false drift on the second deploy when the config uses mixed-case names. +resources: + schemas: + foo: + catalog_name: main + name: MySCHEMA-$UNIQUE_NAME + + volumes: + bar: + catalog_name: main + schema_name: MySCHEMA-$UNIQUE_NAME + name: my-vol-$UNIQUE_NAME + volume_type: MANAGED diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 023feb47cdc..44ff0422b89 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -33,6 +33,7 @@ EnvMatrix.INPUT_CONFIG = [ "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 023feb47cdc..44ff0422b89 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -33,6 +33,7 @@ EnvMatrix.INPUT_CONFIG = [ "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 023feb47cdc..44ff0422b89 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -33,6 +33,7 @@ EnvMatrix.INPUT_CONFIG = [ "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 6b0ee25fc61..a54cc8110ed 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -51,6 +51,7 @@ EnvMatrix.INPUT_CONFIG = [ "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", + "schema_uppercase_name.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", From 30be520660bb8837a881d3ac2cbc7342b9ed41c2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 10 Jun 2026 12:15:17 +0200 Subject: [PATCH 2/7] Make UC case/slash normalization declarative in resources.yml The direct engine saves local config to state, so when the UC API normalizes an identifier (lowercases catalog/schema/volume names) or strips a trailing slash from a storage URL, the next plan compares the original local value against the normalized remote value and triggers a spurious recreate/update. On every second deploy with no config changes, schemas and volumes were recreated/renamed. Previously this was suppressed with per-resource OverrideChangeDesc methods. Replace those with two declarative resources.yml keys, normalize_case and normalize_slash, parsed at runtime like the other lifecycle rules and applied in addPerFieldActions before recreate/update classification. Adding a new UC resource is now a few YAML lines instead of a Go method. - config.go: NormalizeCase / NormalizeSlash on ResourceLifecycleConfig - bundle_plan.go: shouldSkipNormalized wired into the skip-ladder - schema.go, volume.go: delete OverrideChangeDesc; declare fields in resources.yml - testserver: lowercase schema/volume identifier names to mimic UC - schema_uppercase_name invariant config is schema-only (the migrate invariant deploys via Terraform first, and the TF provider rejects an uppercase volume schema_name); volume coverage lives in resources/volumes/uppercase-name Co-authored-by: Isaac --- .../configs/schema_uppercase_name.yml.tmpl | 11 +++---- .../schemas/uppercase-name/databricks.yml | 8 +++++ .../schemas/uppercase-name/out.test.toml | 4 +++ .../schemas/uppercase-name/output.txt | 28 ++++++++++++++++ .../resources/schemas/uppercase-name/script | 10 ++++++ .../schemas/uppercase-name/test.toml | 6 ++++ .../volumes/uppercase-name/databricks.yml | 15 +++++++++ .../uppercase-name/out.deploy.direct.txt | 32 +++++++++++++++++++ .../uppercase-name/out.deploy.terraform.txt | 27 ++++++++++++++++ .../volumes/uppercase-name/out.test.toml | 4 +++ .../volumes/uppercase-name/output.txt | 0 .../resources/volumes/uppercase-name/script | 14 ++++++++ .../volumes/uppercase-name/test.toml | 11 +++++++ bundle/deployplan/plan.go | 1 + bundle/direct/bundle_plan.go | 29 +++++++++++++++++ bundle/direct/dresources/config.go | 10 ++++++ bundle/direct/dresources/resources.yml | 23 +++++++++++++ bundle/direct/dresources/volume.go | 31 ------------------ libs/testserver/schemas.go | 3 ++ libs/testserver/volumes.go | 3 ++ 20 files changed, 232 insertions(+), 38 deletions(-) create mode 100644 acceptance/bundle/resources/schemas/uppercase-name/databricks.yml create mode 100644 acceptance/bundle/resources/schemas/uppercase-name/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/uppercase-name/output.txt create mode 100644 acceptance/bundle/resources/schemas/uppercase-name/script create mode 100644 acceptance/bundle/resources/schemas/uppercase-name/test.toml create mode 100644 acceptance/bundle/resources/volumes/uppercase-name/databricks.yml create mode 100644 acceptance/bundle/resources/volumes/uppercase-name/out.deploy.direct.txt create mode 100644 acceptance/bundle/resources/volumes/uppercase-name/out.deploy.terraform.txt create mode 100644 acceptance/bundle/resources/volumes/uppercase-name/out.test.toml create mode 100644 acceptance/bundle/resources/volumes/uppercase-name/output.txt create mode 100644 acceptance/bundle/resources/volumes/uppercase-name/script create mode 100644 acceptance/bundle/resources/volumes/uppercase-name/test.toml diff --git a/acceptance/bundle/invariant/configs/schema_uppercase_name.yml.tmpl b/acceptance/bundle/invariant/configs/schema_uppercase_name.yml.tmpl index 46e59e26bc9..abf993785f2 100644 --- a/acceptance/bundle/invariant/configs/schema_uppercase_name.yml.tmpl +++ b/acceptance/bundle/invariant/configs/schema_uppercase_name.yml.tmpl @@ -3,15 +3,12 @@ bundle: # Reproduce: UC API normalizes identifier names to lowercase, causing # false drift on the second deploy when the config uses mixed-case names. +# Volumes are covered by acceptance/bundle/resources/volumes/uppercase-name +# instead: the Terraform provider rejects an uppercase volume schema_name +# ("inconsistent final plan"), and the migrate invariant always deploys via +# Terraform first, so an uppercase volume cannot live in this shared config. resources: schemas: foo: catalog_name: main name: MySCHEMA-$UNIQUE_NAME - - volumes: - bar: - catalog_name: main - schema_name: MySCHEMA-$UNIQUE_NAME - name: my-vol-$UNIQUE_NAME - volume_type: MANAGED diff --git a/acceptance/bundle/resources/schemas/uppercase-name/databricks.yml b/acceptance/bundle/resources/schemas/uppercase-name/databricks.yml new file mode 100644 index 00000000000..462149c4867 --- /dev/null +++ b/acceptance/bundle/resources/schemas/uppercase-name/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test-bundle + +resources: + schemas: + schema1: + catalog_name: main + name: MySCHEMA diff --git a/acceptance/bundle/resources/schemas/uppercase-name/out.test.toml b/acceptance/bundle/resources/schemas/uppercase-name/out.test.toml new file mode 100644 index 00000000000..e849ec85ace --- /dev/null +++ b/acceptance/bundle/resources/schemas/uppercase-name/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/uppercase-name/output.txt b/acceptance/bundle/resources/schemas/uppercase-name/output.txt new file mode 100644 index 00000000000..6f8d9b7f9b4 --- /dev/null +++ b/acceptance/bundle/resources/schemas/uppercase-name/output.txt @@ -0,0 +1,28 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "MySCHEMA" + } +} + +=== Bundle summary shows lowercase ID after UC normalizes the name +"main.myschema" + +=== Redeploy without changes - should be a no-op +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/uppercase-name/script b/acceptance/bundle/resources/schemas/uppercase-name/script new file mode 100644 index 00000000000..5f3241f9127 --- /dev/null +++ b/acceptance/bundle/resources/schemas/uppercase-name/script @@ -0,0 +1,10 @@ +echo "*" > .gitignore +trace $CLI bundle deploy +trace print_requests.py //unity-catalog + +title "Bundle summary shows lowercase ID after UC normalizes the name\n" +$CLI bundle summary -o json | jq .resources.schemas.schema1.id + +title "Redeploy without changes - should be a no-op" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/uppercase-name/test.toml b/acceptance/bundle/resources/schemas/uppercase-name/test.toml new file mode 100644 index 00000000000..6d3d4f08d22 --- /dev/null +++ b/acceptance/bundle/resources/schemas/uppercase-name/test.toml @@ -0,0 +1,6 @@ +Cloud = true +RequiresUnityCatalog = true + +Ignore = [ + ".databricks", +] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/databricks.yml b/acceptance/bundle/resources/volumes/uppercase-name/databricks.yml new file mode 100644 index 00000000000..3a8448bddda --- /dev/null +++ b/acceptance/bundle/resources/volumes/uppercase-name/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +resources: + schemas: + schema1: + catalog_name: main + name: MySCHEMA + + volumes: + vol1: + catalog_name: main + schema_name: MySCHEMA + name: MyVOLUME + volume_type: MANAGED diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.direct.txt b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.direct.txt new file mode 100644 index 00000000000..eb184f8d69e --- /dev/null +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.direct.txt @@ -0,0 +1,32 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "MySCHEMA" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/volumes", + "body": { + "catalog_name": "main", + "name": "MyVOLUME", + "schema_name": "MySCHEMA", + "volume_type": "MANAGED" + } +} +=== Bundle summary shows lowercase IDs after UC normalizes names +{ + "schema_id": "main.myschema", + "volume_id": "main.myschema.myvolume" +} +=== Redeploy without changes - should be a no-op +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.terraform.txt b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.terraform.txt new file mode 100644 index 00000000000..2cd89623003 --- /dev/null +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.terraform.txt @@ -0,0 +1,27 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Error: terraform apply: exit status 1 + +Error: Provider produced inconsistent final plan + +When expanding the plan for databricks_volume.vol1 to include new values +learned so far during apply, provider +"registry.terraform.io/databricks/databricks" produced an invalid new value +for .schema_name: was cty.StringVal("MySCHEMA"), but now +cty.StringVal("myschema"). + +This is a bug in the provider, which should be reported in the provider's own +issue tracker. + + +Updating deployment state... + +Exit code: 1 +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "main", + "name": "MySCHEMA" + } +} diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml new file mode 100644 index 00000000000..e849ec85ace --- /dev/null +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/output.txt b/acceptance/bundle/resources/volumes/uppercase-name/output.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/volumes/uppercase-name/script b/acceptance/bundle/resources/volumes/uppercase-name/script new file mode 100644 index 00000000000..023d1d0f723 --- /dev/null +++ b/acceptance/bundle/resources/volumes/uppercase-name/script @@ -0,0 +1,14 @@ +echo "*" > .gitignore +errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +print_requests.py //unity-catalog >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + +if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then + echo "=== Bundle summary shows lowercase IDs after UC normalizes names" >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + $CLI bundle summary -o json \ + | jq '{schema_id: .resources.schemas.schema1.id, volume_id: .resources.volumes.vol1.id}' \ + >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + + echo "=== Redeploy without changes - should be a no-op" >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + errcode $CLI bundle deploy &>> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + print_requests.py //unity-catalog >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +fi diff --git a/acceptance/bundle/resources/volumes/uppercase-name/test.toml b/acceptance/bundle/resources/volumes/uppercase-name/test.toml new file mode 100644 index 00000000000..8331e99f292 --- /dev/null +++ b/acceptance/bundle/resources/volumes/uppercase-name/test.toml @@ -0,0 +1,11 @@ +Cloud = true +RequiresUnityCatalog = true + +Ignore = [ + ".databricks", +] + +# TF provider fails with "inconsistent final plan" when schema_name contains +# uppercase letters that UC normalizes to lowercase; exclude from cloud runs. +[EnvMatrixExclude] +no_terraform_on_cloud = ["CONFIG_Cloud=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 2fb5d38c806..3ff9fe2cf1f 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -101,6 +101,7 @@ const ( ReasonBackendDefault = "backend_default" ReasonAlias = "alias" ReasonURLNormalization = "url_normalization" + ReasonUCNormalization = "uc_normalization" ReasonRemoteAlreadySet = "remote_already_set" ReasonEmpty = "empty" ReasonCustom = "custom" diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5591626cd75..346b37392eb 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -375,6 +375,12 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change } else if reason, ok := shouldSkipBackendDefault(generatedCfg, path, ch); ok { ch.Action = deployplan.Skip ch.Reason = reason + } else if reason, ok := shouldSkipNormalized(cfg, path, ch); ok { + ch.Action = deployplan.Skip + ch.Reason = reason + } else if reason, ok := shouldSkipNormalized(generatedCfg, path, ch); ok { + ch.Action = deployplan.Skip + ch.Reason = reason } else if action, reason := shouldUpdateOrRecreate(cfg, path); action != deployplan.Undefined { ch.Action = action ch.Reason = reason @@ -434,6 +440,29 @@ func shouldSkip(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNo return "", false } +// shouldSkipNormalized skips a change that is a false diff caused by UC API +// normalization: the API lowercases identifier names (normalize_case) and strips +// trailing slashes from storage URLs (normalize_slash). The direct engine saves +// local config to state, so without this the next plan sees the original value +// against the normalized remote value and triggers a spurious recreate/update. +func shouldSkipNormalized(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, ch *deployplan.ChangeDesc) (string, bool) { + if cfg == nil { + return "", false + } + newStr, newOk := ch.New.(string) + remoteStr, remoteOk := ch.Remote.(string) + if !newOk || !remoteOk { + return "", false + } + if _, ok := findMatchingRule(path, cfg.NormalizeCase); ok && strings.EqualFold(newStr, remoteStr) { + return deployplan.ReasonUCNormalization, true + } + if _, ok := findMatchingRule(path, cfg.NormalizeSlash); ok && strings.TrimRight(newStr, "/") == strings.TrimRight(remoteStr, "/") { + return deployplan.ReasonURLNormalization, true + } + return "", false +} + func shouldUpdateOrRecreate(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode) (deployplan.ActionType, string) { if cfg == nil { return deployplan.Undefined, "" diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 70f6dbb1d8d..a425d7d7208 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -59,6 +59,14 @@ type ResourceLifecycleConfig struct { // UpdateIDOnChanges: field patterns that trigger UpdateWithID when changed. UpdateIDOnChanges []FieldRule `yaml:"update_id_on_changes,omitempty"` + // NormalizeCase: string field patterns the UC API lowercases on write. + // A change is skipped when local and remote differ only by case. + NormalizeCase []FieldRule `yaml:"normalize_case,omitempty"` + + // NormalizeSlash: string field patterns the UC API strips trailing slashes from. + // A change is skipped when local and remote differ only by trailing slashes. + NormalizeSlash []FieldRule `yaml:"normalize_slash,omitempty"` + // BackendDefaults: fields where the backend may set defaults. // When old and new are nil but remote is set, and the remote value matches allowed values (if specified), the change is skipped. BackendDefaults []BackendDefaultRule `yaml:"backend_defaults,omitempty"` @@ -80,6 +88,8 @@ var empty = ResourceLifecycleConfig{ IgnoreLocalChanges: nil, RecreateOnChanges: nil, UpdateIDOnChanges: nil, + NormalizeCase: nil, + NormalizeSlash: nil, BackendDefaults: nil, } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 75a60e0dc62..b69c751224e 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -311,6 +311,15 @@ resources: reason: immutable - field: storage_root reason: immutable + normalize_case: + # UC lowercases identifier names; remote returns "myschema" for config "MySchema". + - field: name + reason: uc_identifier + - field: catalog_name + reason: uc_identifier + normalize_slash: + - field: storage_root + reason: uc_strips_trailing_slash external_locations: recreate_on_changes: @@ -341,6 +350,20 @@ resources: update_id_on_changes: - field: name reason: id_changes + normalize_case: + # UC lowercases identifier names. name is in update_id_on_changes, so a + # case-only diff would otherwise trigger a spurious rename (DoUpdateWithID). + - field: catalog_name + reason: uc_identifier + - field: schema_name + reason: uc_identifier + - field: name + reason: uc_identifier + normalize_slash: + # UC strips trailing slashes on create; matches the Terraform provider's suppressLocationDiff. + # https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25 + - field: storage_location + reason: uc_strips_trailing_slash backend_defaults: # storage_location is Computed; backend generates it for managed volumes. - field: storage_location diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 73bf7a79b40..6c96e66eccb 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -6,9 +6,7 @@ import ( "strings" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -114,35 +112,6 @@ func (r *ResourceVolume) DoDelete(ctx context.Context, id string, _ *catalog.Cre return r.client.Volumes.DeleteByName(ctx, id) } -// OverrideChangeDesc suppresses drift for storage_location when the only difference -// is a trailing slash. The UC API strips trailing slashes on create, so remote returns -// "s3://bucket/path" while the config may have "s3://bucket/path/". -// -// This matches the Terraform provider's suppressLocationDiff behavior. -// https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25 -func (*ResourceVolume) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *catalog.VolumeInfo) error { - if change.Action == deployplan.Skip { - return nil - } - - if path.String() != "storage_location" { - return nil - } - - newStr, newOk := change.New.(string) - remoteStr, remoteOk := change.Remote.(string) - if !newOk || !remoteOk { - return nil - } - - if newStr != remoteStr && strings.TrimRight(newStr, "/") == strings.TrimRight(remoteStr, "/") { - change.Action = deployplan.Skip - change.Reason = deployplan.ReasonURLNormalization - } - - return nil -} - func getNameFromID(id string) (string, error) { items := strings.Split(id, ".") if len(items) == 0 { diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 1d4dc79e7ac..fa190c6694a 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "dario.cat/mergo" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -23,6 +24,8 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { } } + // UC normalizes schema names to lowercase. + schema.Name = strings.ToLower(schema.Name) schema.FullName = schema.CatalogName + "." + schema.Name schema.ForceSendFields = []string{"BrowseOnly"} schema.CatalogType = "MANAGED_CATALOG" diff --git a/libs/testserver/volumes.go b/libs/testserver/volumes.go index 76b5fdf3e5f..88eae7ac021 100644 --- a/libs/testserver/volumes.go +++ b/libs/testserver/volumes.go @@ -20,6 +20,9 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { } } + // UC normalizes schema and volume names to lowercase. + volume.SchemaName = strings.ToLower(volume.SchemaName) + volume.Name = strings.ToLower(volume.Name) volume.FullName = volume.CatalogName + "." + volume.SchemaName + "." + volume.Name if volume.StorageLocation != "" && volume.VolumeType != catalog.VolumeTypeExternal { From e59cc1c15ff3e0aadd07c283cb526d997ed5901f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 10 Jun 2026 12:39:39 +0200 Subject: [PATCH 3/7] Rename normalize_case reason to uc_case Co-authored-by: Isaac --- bundle/direct/dresources/resources.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index b69c751224e..b932ffd4286 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -314,9 +314,9 @@ resources: normalize_case: # UC lowercases identifier names; remote returns "myschema" for config "MySchema". - field: name - reason: uc_identifier + reason: uc_case - field: catalog_name - reason: uc_identifier + reason: uc_case normalize_slash: - field: storage_root reason: uc_strips_trailing_slash @@ -354,11 +354,11 @@ resources: # UC lowercases identifier names. name is in update_id_on_changes, so a # case-only diff would otherwise trigger a spurious rename (DoUpdateWithID). - field: catalog_name - reason: uc_identifier + reason: uc_case - field: schema_name - reason: uc_identifier + reason: uc_case - field: name - reason: uc_identifier + reason: uc_case normalize_slash: # UC strips trailing slashes on create; matches the Terraform provider's suppressLocationDiff. # https://github.com/databricks/terraform-provider-databricks/blob/v1.65.1/catalog/resource_volume.go#L25 From 4ea89e4f0d23566e2cf31e88253d1c2241d81583 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 10 Jun 2026 12:54:54 +0200 Subject: [PATCH 4/7] acc: add volume_uppercase_name invariant, drop redundant schema test volume_uppercase_name covers the uppercase no-drift case on direct via the no_drift invariant; excluded from migrate (Terraform deploys first and rejects an uppercase volume schema_name). The dedicated schemas/uppercase-name test is redundant with the no_drift invariant, so remove it. Co-authored-by: Isaac --- .../configs/volume_uppercase_name.yml.tmpl | 22 +++++++++++++++ .../invariant/continue_293/out.test.toml | 3 +- .../bundle/invariant/migrate/out.test.toml | 3 +- acceptance/bundle/invariant/migrate/test.toml | 4 +++ .../bundle/invariant/no_drift/out.test.toml | 3 +- acceptance/bundle/invariant/test.toml | 1 + .../schemas/uppercase-name/databricks.yml | 8 ------ .../schemas/uppercase-name/out.test.toml | 4 --- .../schemas/uppercase-name/output.txt | 28 ------------------- .../resources/schemas/uppercase-name/script | 10 ------- .../schemas/uppercase-name/test.toml | 6 ---- 11 files changed, 33 insertions(+), 59 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/volume_uppercase_name.yml.tmpl delete mode 100644 acceptance/bundle/resources/schemas/uppercase-name/databricks.yml delete mode 100644 acceptance/bundle/resources/schemas/uppercase-name/out.test.toml delete mode 100644 acceptance/bundle/resources/schemas/uppercase-name/output.txt delete mode 100644 acceptance/bundle/resources/schemas/uppercase-name/script delete mode 100644 acceptance/bundle/resources/schemas/uppercase-name/test.toml diff --git a/acceptance/bundle/invariant/configs/volume_uppercase_name.yml.tmpl b/acceptance/bundle/invariant/configs/volume_uppercase_name.yml.tmpl new file mode 100644 index 00000000000..c6caa55caa7 --- /dev/null +++ b/acceptance/bundle/invariant/configs/volume_uppercase_name.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +# Reproduce: the UC API lowercases identifier names, causing false drift on the +# second deploy when the config uses mixed-case names. This covers a volume's +# catalog_name, schema_name and name on the direct engine. +# +# Excluded from the migrate invariant: that test deploys via Terraform first, +# and the TF provider rejects an uppercase volume schema_name ("inconsistent +# final plan"). +resources: + schemas: + foo: + catalog_name: main + name: MySCHEMA-$UNIQUE_NAME + + volumes: + bar: + catalog_name: main + schema_name: ${resources.schemas.foo.name} + name: MyVOL-$UNIQUE_NAME + volume_type: MANAGED diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 44ff0422b89..86ca885dc35 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -43,5 +43,6 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_endpoint.yml.tmpl", "vector_search_index.yml.tmpl", "volume.yml.tmpl", - "volume_external.yml.tmpl" + "volume_external.yml.tmpl", + "volume_uppercase_name.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 44ff0422b89..86ca885dc35 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -43,5 +43,6 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_endpoint.yml.tmpl", "vector_search_index.yml.tmpl", "volume.yml.tmpl", - "volume_external.yml.tmpl" + "volume_external.yml.tmpl", + "volume_uppercase_name.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 5fa381832e7..617f144e95d 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -24,3 +24,7 @@ EnvMatrixExclude.no_sql_warehouse = ["INPUT_CONFIG=sql_warehouse.yml.tmpl"] # The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min # per variant (deploy + migrate + plan at 1000 tasks) without incremental coverage. EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] + +# migrate deploys via Terraform first, and the TF provider rejects an uppercase +# volume schema_name ("inconsistent final plan"). Covered by no_drift on direct. +EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 44ff0422b89..86ca885dc35 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -43,5 +43,6 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_endpoint.yml.tmpl", "vector_search_index.yml.tmpl", "volume.yml.tmpl", - "volume_external.yml.tmpl" + "volume_external.yml.tmpl", + "volume_uppercase_name.yml.tmpl" ] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index a54cc8110ed..2751a6f5a2b 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -62,6 +62,7 @@ EnvMatrix.INPUT_CONFIG = [ "vector_search_index.yml.tmpl", "volume.yml.tmpl", "volume_external.yml.tmpl", + "volume_uppercase_name.yml.tmpl", ] [EnvMatrixExclude] diff --git a/acceptance/bundle/resources/schemas/uppercase-name/databricks.yml b/acceptance/bundle/resources/schemas/uppercase-name/databricks.yml deleted file mode 100644 index 462149c4867..00000000000 --- a/acceptance/bundle/resources/schemas/uppercase-name/databricks.yml +++ /dev/null @@ -1,8 +0,0 @@ -bundle: - name: test-bundle - -resources: - schemas: - schema1: - catalog_name: main - name: MySCHEMA diff --git a/acceptance/bundle/resources/schemas/uppercase-name/out.test.toml b/acceptance/bundle/resources/schemas/uppercase-name/out.test.toml deleted file mode 100644 index e849ec85ace..00000000000 --- a/acceptance/bundle/resources/schemas/uppercase-name/out.test.toml +++ /dev/null @@ -1,4 +0,0 @@ -Local = true -Cloud = true -RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/uppercase-name/output.txt b/acceptance/bundle/resources/schemas/uppercase-name/output.txt deleted file mode 100644 index 6f8d9b7f9b4..00000000000 --- a/acceptance/bundle/resources/schemas/uppercase-name/output.txt +++ /dev/null @@ -1,28 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> print_requests.py //unity-catalog -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/schemas", - "body": { - "catalog_name": "main", - "name": "MySCHEMA" - } -} - -=== Bundle summary shows lowercase ID after UC normalizes the name -"main.myschema" - -=== Redeploy without changes - should be a no-op ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/uppercase-name/script b/acceptance/bundle/resources/schemas/uppercase-name/script deleted file mode 100644 index 5f3241f9127..00000000000 --- a/acceptance/bundle/resources/schemas/uppercase-name/script +++ /dev/null @@ -1,10 +0,0 @@ -echo "*" > .gitignore -trace $CLI bundle deploy -trace print_requests.py //unity-catalog - -title "Bundle summary shows lowercase ID after UC normalizes the name\n" -$CLI bundle summary -o json | jq .resources.schemas.schema1.id - -title "Redeploy without changes - should be a no-op" -trace $CLI bundle deploy -trace print_requests.py //unity-catalog diff --git a/acceptance/bundle/resources/schemas/uppercase-name/test.toml b/acceptance/bundle/resources/schemas/uppercase-name/test.toml deleted file mode 100644 index 6d3d4f08d22..00000000000 --- a/acceptance/bundle/resources/schemas/uppercase-name/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -Cloud = true -RequiresUnityCatalog = true - -Ignore = [ - ".databricks", -] From 9df55b378365a0245892c19a72e8b7e54b906283 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 10 Jun 2026 12:55:50 +0200 Subject: [PATCH 5/7] Update NEXT_CHANGELOG.md Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 90b65ec3e86..5df236fa7e4 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,6 +9,7 @@ ### Bundles * Set the default `data_security_mode` to `DATA_SECURITY_MODE_AUTO` in bundle templates ([#5452](https://github.com/databricks/cli/pull/5452)). * Mark vector search index index_subtype as backend_default to prevent drift after deployment ([#5454](https://github.com/databricks/cli/pull/5454)). +* Fix spurious recreate of schemas and volumes whose names use mixed case ([#5531](https://github.com/databricks/cli/pull/5531)). ### Dependency updates From 6216b372e7ce63de3b1b17a78bd0fa7efe107fd2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 10 Jun 2026 14:21:51 +0200 Subject: [PATCH 6/7] Use config reason for normalize_case/normalize_slash skips shouldSkipNormalized returned a hardcoded reason and ignored the per-field reason from resources.yml, unlike every other rule. Route the config reason through (like recreate_on_changes etc.) and drop the now-unused Reason{UC,URL}Normalization constants. Record the relevant plan changes (field action + reason) in the volumes/uppercase-name test so the skip classification is visible. Co-authored-by: Isaac --- .../uppercase-name/out.deploy.direct.txt | 29 +++++++++++++++++++ .../resources/volumes/uppercase-name/script | 5 ++++ bundle/deployplan/plan.go | 2 -- bundle/direct/bundle_plan.go | 8 ++--- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.direct.txt b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.direct.txt index eb184f8d69e..440c0314386 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.direct.txt +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.deploy.direct.txt @@ -25,6 +25,35 @@ Deployment complete! "schema_id": "main.myschema", "volume_id": "main.myschema.myvolume" } +=== Plan on no-op redeploy: mixed-case names are skipped, not recreated +[ + { + "resource": "resources.schemas.schema1", + "changes": { + "name": { + "action": "skip", + "reason": "uc_case" + } + } + }, + { + "resource": "resources.volumes.vol1", + "changes": { + "name": { + "action": "skip", + "reason": "uc_case" + }, + "schema_name": { + "action": "skip", + "reason": "uc_case" + }, + "storage_location": { + "action": "skip", + "reason": "backend_default" + } + } + } +] === Redeploy without changes - should be a no-op Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... diff --git a/acceptance/bundle/resources/volumes/uppercase-name/script b/acceptance/bundle/resources/volumes/uppercase-name/script index 023d1d0f723..23ea36eca82 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/script +++ b/acceptance/bundle/resources/volumes/uppercase-name/script @@ -8,6 +8,11 @@ if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then | jq '{schema_id: .resources.schemas.schema1.id, volume_id: .resources.volumes.vol1.id}' \ >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + echo "=== Plan on no-op redeploy: mixed-case names are skipped, not recreated" >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + $CLI bundle plan -o json \ + | jq '.plan | to_entries | map({resource: .key, changes: (.value.changes // {} | map_values({action, reason}))})' \ + >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + echo "=== Redeploy without changes - should be a no-op" >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt errcode $CLI bundle deploy &>> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt print_requests.py //unity-catalog >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 3ff9fe2cf1f..6254e92b802 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -100,8 +100,6 @@ type ChangeDesc struct { const ( ReasonBackendDefault = "backend_default" ReasonAlias = "alias" - ReasonURLNormalization = "url_normalization" - ReasonUCNormalization = "uc_normalization" ReasonRemoteAlreadySet = "remote_already_set" ReasonEmpty = "empty" ReasonCustom = "custom" diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 346b37392eb..fea24b79808 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -454,11 +454,11 @@ func shouldSkipNormalized(cfg *dresources.ResourceLifecycleConfig, path *structp if !newOk || !remoteOk { return "", false } - if _, ok := findMatchingRule(path, cfg.NormalizeCase); ok && strings.EqualFold(newStr, remoteStr) { - return deployplan.ReasonUCNormalization, true + if reason, ok := findMatchingRule(path, cfg.NormalizeCase); ok && strings.EqualFold(newStr, remoteStr) { + return reason, true } - if _, ok := findMatchingRule(path, cfg.NormalizeSlash); ok && strings.TrimRight(newStr, "/") == strings.TrimRight(remoteStr, "/") { - return deployplan.ReasonURLNormalization, true + if reason, ok := findMatchingRule(path, cfg.NormalizeSlash); ok && strings.TrimRight(newStr, "/") == strings.TrimRight(remoteStr, "/") { + return reason, true } return "", false } From 858eacf96f03673c011c41c8979b1ce7c11891ce Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 10 Jun 2026 16:55:09 +0200 Subject: [PATCH 7/7] Fix bash 3.2 syntax error in volumes/uppercase-name script &>> is bash 4+ syntax; macOS CI runners use the stock bash 3.2, which fails to parse the whole script ("syntax error near unexpected token '>'") and produces no output files. Use >> ... 2>&1 instead. Co-authored-by: Isaac --- acceptance/bundle/resources/volumes/uppercase-name/script | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/volumes/uppercase-name/script b/acceptance/bundle/resources/volumes/uppercase-name/script index 23ea36eca82..e8521397d3e 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/script +++ b/acceptance/bundle/resources/volumes/uppercase-name/script @@ -14,6 +14,7 @@ if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt echo "=== Redeploy without changes - should be a no-op" >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt - errcode $CLI bundle deploy &>> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt + # &>> requires bash 4+; macOS CI runs the stock bash 3.2, so use >> ... 2>&1. + errcode $CLI bundle deploy >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 print_requests.py //unity-catalog >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt fi