[Internal] Rebase and Fixed merge conflicts with main branch. Also fixed notification_destination acceptance tests.

.github/workflows/message.yml

@@ -3,20 +3,27 @@ name: Validate Commit Message
 on:
   pull_request:
     types: [opened, synchronize, edited]
+  merge_group:
+      types: [checks_requested] 
 
 jobs:
-
   validate:
     runs-on: ubuntu-latest
+    # GitHub required checks are shared between PRs and the Merge Queue.
+    # Since there is no PR title on Merge Queue, we need to trigger and
+    # skip this test for Merge Queue to succeed.
+    if: github.event_name == 'pull_request' 
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Validate Tag
+        env:
+          TITLE: ${{ github.event.pull_request.title }}
         run: |
-          TAG=$(echo ${{ github.event.pull_request.title }} | sed -ne 's/\[\(.*\)\].*/\1/p')
+          TAG=$(echo "$TITLE" | sed -ne 's/\[\(.*\)\].*/\1/p')
           if grep -q "tag: \"\[$TAG\]\"" .codegen/changelog_config.yml; then 
             echo "Valid tag found: [$TAG]"
           else 

CHANGELOG.md

@@ -1,5 +1,78 @@
 # Version changelog
 
+## 1.49.1
+
+### Bug Fixes
+ * Fixed reading of permissions for SQL objects ([#3800](https://github.com/databricks/terraform-provider-databricks/pull/3800)).
+ * don't update `databricks_metastore` during creation if not required ([#3783](https://github.com/databricks/terraform-provider-databricks/pull/3783)).
+
+### Documentation
+ * Clarified schedule block in `databricks_job` ([#3805](https://github.com/databricks/terraform-provider-databricks/pull/3805)).
+ * Use correct names for isolation mode for storage credentials and external locations ([#3804](https://github.com/databricks/terraform-provider-databricks/pull/3804)).
+ * Fix incomplete note in databricks_workspace_binding resource ([#3806](https://github.com/databricks/terraform-provider-databricks/pull/3806))
+
+### Internal Changes
+ * Refactored `databricks_zones` and `databricks_spark_versions` data sources to Go SDK ([#3687](https://github.com/databricks/terraform-provider-databricks/pull/3687)).
+
+### Exporter
+ * Add support for exporting of Lakeview dashboards ([#3779](https://github.com/databricks/terraform-provider-databricks/pull/3779)).
+ * Adding more retries for SCIM API calls ([#3807](https://github.com/databricks/terraform-provider-databricks/pull/3807))
+
+
+## 1.49.0
+
+### New Features and Improvements
+ * Added `databricks_dashboard` resource ([#3729](https://github.com/databricks/terraform-provider-databricks/pull/3729)).
+ * Added `databricks_schema` data source ([#3732](https://github.com/databricks/terraform-provider-databricks/pull/3732)).
+ * Added support for binding storage credentials and external locations to specific workspaces ([#3678](https://github.com/databricks/terraform-provider-databricks/pull/3678)).
+ * Added `databricks_volume` as data source  ([#3211](https://github.com/databricks/terraform-provider-databricks/pull/3211)).
+ * Make the `schedule.pause_status` field read-only ([#3692](https://github.com/databricks/terraform-provider-databricks/pull/3692)).
+ * Renamed `databricks_catalog_workspace_binding` to `databricks_workspace_binding` ([#3703](https://github.com/databricks/terraform-provider-databricks/pull/3703)).
+ * Make `cluster_name_contains` optional in `databricks_clusters` data source ([#3760](https://github.com/databricks/terraform-provider-databricks/pull/3760)).
+ * Tolerate OAuth errors in databricks_mws_workspaces when managing tokens ([#3761](https://github.com/databricks/terraform-provider-databricks/pull/3761)).
+ * Permissions for `databricks_dashboard` resource ([#3762](https://github.com/databricks/terraform-provider-databricks/pull/3762)).
+ * Fix model serving resource ([#3690](https://github.com/databricks/terraform-provider-databricks/pull/3690))
+
+### Exporter
+ * Emit directories during the listing only if they are explicitly configured in `-listing` ([#3673](https://github.com/databricks/terraform-provider-databricks/pull/3673)).
+ * Export libraries specified as `requirements.txt` ([#3649](https://github.com/databricks/terraform-provider-databricks/pull/3649)).
+ * Fix generation of `run_as` blocks in `databricks_job` ([#3724](https://github.com/databricks/terraform-provider-databricks/pull/3724)).
+ * Use Go SDK structs for `databricks_job` resource ([#3727](https://github.com/databricks/terraform-provider-databricks/pull/3727)).
+ * Clarify use of `-listing` and `-services` options ([#3755](https://github.com/databricks/terraform-provider-databricks/pull/3755)).
+ * Improve code generation for SQL Endpoints ([#3764](https://github.com/databricks/terraform-provider-databricks/pull/3764))
+ * Fix to support Serverless DLT ([#3796](https://github.com/databricks/terraform-provider-databricks/pull/3796))
+
+### Documentation
+ * Fix invalid priviledges in grants.md ([#3716](https://github.com/databricks/terraform-provider-databricks/pull/3716)).
+ * Update cluster.md: add data_security_mode parameters `NONE` and `NO_ISOLATION` ([#3740](https://github.com/databricks/terraform-provider-databricks/pull/3740)).
+ * Remove references to basic auth ([#3720](https://github.com/databricks/terraform-provider-databricks/pull/3720)).
+ * Update resources diagram ([#3765](https://github.com/databricks/terraform-provider-databricks/pull/3765)).
+ * Improve docs for Network Connectivity Config ([#3794](https://github.com/databricks/terraform-provider-databricks/pull/3794))
+ * Document serverless flag in databricks_pipeline ([#3797](https://github.com/databricks/terraform-provider-databricks/pull/3797))
+ * Add description of environment block to databricks_job ([#3798](https://github.com/databricks/terraform-provider-databricks/pull/3798))
+
+
+### Internal Changes
+ * Add Release tag ([#3748](https://github.com/databricks/terraform-provider-databricks/pull/3748)).
+ * Improve Changelog by grouping changes ([#3747](https://github.com/databricks/terraform-provider-databricks/pull/3747)).
+ * Change TF registry ownership ([#3736](https://github.com/databricks/terraform-provider-databricks/pull/3736)).
+ * Refactored `databricks_cluster(s)` data sources to Go SDK ([#3685](https://github.com/databricks/terraform-provider-databricks/pull/3685)).
+ * Upgrade databricks-sdk-go ([#3743](https://github.com/databricks/terraform-provider-databricks/pull/3743)).
+ * Run goreleaser action in snapshot mode from merge queue ([#3646](https://github.com/databricks/terraform-provider-databricks/pull/3646)).
+ * Make `dashboard_name` random in integration tests for `databricks_dashboard` resource ([#3763](https://github.com/databricks/terraform-provider-databricks/pull/3763)).
+ * Clear stale go.sum values ([#3768](https://github.com/databricks/terraform-provider-databricks/pull/3768)).
+ * Add "Owner" tag to test cluster in acceptance test ([#3771](https://github.com/databricks/terraform-provider-databricks/pull/3771)).
+ * Fix integration test for restrict workspace admins setting ([#3772](https://github.com/databricks/terraform-provider-databricks/pull/3772)).
+ * Add "Owner" tag to test SQL endpoint in acceptance test ([#3774](https://github.com/databricks/terraform-provider-databricks/pull/3774)).
+ * Move PR message validation to a separate workflow ([#3777](https://github.com/databricks/terraform-provider-databricks/pull/3777)).
+ * Trigger the validate workflow in the merge queue ([#3782](https://github.com/databricks/terraform-provider-databricks/pull/3782)).
+ * Update properties for managed SQL table on latest DBR ([#3784](https://github.com/databricks/terraform-provider-databricks/pull/3784)).
+ * Add "Owner" tag to test SQL endpoint in acceptance test ([#3785](https://github.com/databricks/terraform-provider-databricks/pull/3785)).
+ * Ignore managed property for liquid clustering integration test ([#3786](https://github.com/databricks/terraform-provider-databricks/pull/3786))
+ * Fix processing of quoted titles ([#3790](https://github.com/databricks/terraform-provider-databricks/pull/3790))
+
+
+
 ## 1.48.3
 
 ### Internal Changes 

access/resource_sql_permissions.go

@@ -272,7 +272,7 @@ func (ta *SqlPermissions) initCluster(ctx context.Context, d *schema.ResourceDat
 }
 
 func (ta *SqlPermissions) getOrCreateCluster(clustersAPI clusters.ClustersAPI) (string, error) {
-	sparkVersion := clustersAPI.LatestSparkVersionOrDefault(clusters.SparkVersionRequest{
+	sparkVersion := clusters.LatestSparkVersionOrDefault(clustersAPI.Context(), clustersAPI.WorkspaceClient(), compute.SparkVersionRequest{
 		Latest: true,
 	})
 	nodeType := clustersAPI.GetSmallestNodeType(compute.NodeTypeRequest{LocalDisk: true})

access/resource_sql_permissions_test.go

@@ -184,20 +184,20 @@ var createHighConcurrencyCluster = []qa.HTTPFixture{
 	{
 		Method:       "GET",
 		ReuseRequest: true,
-		Resource:     "/api/2.0/clusters/spark-versions",
-		Response: clusters.SparkVersionsList{
-			SparkVersions: []clusters.SparkVersion{
+		Resource:     "/api/2.1/clusters/spark-versions",
+		Response: compute.GetSparkVersionsResponse{
+			Versions: []compute.SparkVersion{
 				{
-					Version:     "7.1.x-cpu-ml-scala2.12",
-					Description: "7.1 ML (includes Apache Spark 3.0.0, Scala 2.12)",
+					Key:  "7.1.x-cpu-ml-scala2.12",
+					Name: "7.1 ML (includes Apache Spark 3.0.0, Scala 2.12)",
 				},
 			},
 		},
 	},
 	{
 		Method:       "GET",
 		ReuseRequest: true,
-		Resource:     "/api/2.0/clusters/list-node-types",
+		Resource:     "/api/2.1/clusters/list-node-types",
 		Response: compute.ListNodeTypesResponse{
 			NodeTypes: []compute.NodeType{
 				{
@@ -222,7 +222,7 @@ var createHighConcurrencyCluster = []qa.HTTPFixture{
 			AutoterminationMinutes: 10,
 			ClusterName:            "terraform-table-acl",
 			NodeTypeID:             "Standard_F4s",
-			SparkVersion:           "7.3.x-scala2.12",
+			SparkVersion:           "11.3.x-scala2.12",
 			CustomTags: map[string]string{
 				"ResourceClass": "SingleNode",
 			},
@@ -261,20 +261,20 @@ var createSharedCluster = []qa.HTTPFixture{
 	{
 		Method:       "GET",
 		ReuseRequest: true,
-		Resource:     "/api/2.0/clusters/spark-versions",
-		Response: clusters.SparkVersionsList{
-			SparkVersions: []clusters.SparkVersion{
+		Resource:     "/api/2.1/clusters/spark-versions",
+		Response: compute.GetSparkVersionsResponse{
+			Versions: []compute.SparkVersion{
 				{
-					Version:     "7.1.x-cpu-ml-scala2.12",
-					Description: "7.1 ML (includes Apache Spark 3.0.0, Scala 2.12)",
+					Key:  "7.1.x-cpu-ml-scala2.12",
+					Name: "7.1 ML (includes Apache Spark 3.0.0, Scala 2.12)",
 				},
 			},
 		},
 	},
 	{
 		Method:       "GET",
 		ReuseRequest: true,
-		Resource:     "/api/2.0/clusters/list-node-types",
+		Resource:     "/api/2.1/clusters/list-node-types",
 		Response: compute.ListNodeTypesResponse{
 			NodeTypes: []compute.NodeType{
 				{
@@ -299,7 +299,7 @@ var createSharedCluster = []qa.HTTPFixture{
 			AutoterminationMinutes: 10,
 			ClusterName:            "terraform-table-acl",
 			NodeTypeID:             "Standard_F4s",
-			SparkVersion:           "7.3.x-scala2.12",
+			SparkVersion:           "11.3.x-scala2.12",
 			CustomTags: map[string]string{
 				"ResourceClass": "SingleNode",
 			},

aws/data_aws_unity_catalog_policy.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"regexp"
+	"strings"
 
 	"github.com/databricks/terraform-provider-databricks/common"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
@@ -44,16 +45,18 @@ func generateReadContext(ctx context.Context, d *schema.ResourceData, m *common.
 		},
 	}
 	if kmsKey, ok := d.GetOk("kms_name"); ok {
+		kmsArn := fmt.Sprintf("arn:aws:kms:%s", kmsKey)
+		if strings.HasPrefix(kmsKey.(string), "arn:aws") {
+			kmsArn = kmsKey.(string)
+		}
 		policy.Statements = append(policy.Statements, &awsIamPolicyStatement{
 			Effect: "Allow",
 			Actions: []string{
 				"kms:Decrypt",
 				"kms:Encrypt",
 				"kms:GenerateDataKey*",
 			},
-			Resources: []string{
-				fmt.Sprintf("arn:aws:kms:%s", kmsKey),
-			},
+			Resources: []string{kmsArn},
 		})
 	}
 	policyJSON, err := json.MarshalIndent(policy, "", "  ")
@@ -73,9 +76,6 @@ func validateSchema() map[string]*schema.Schema {
 		"kms_name": {
 			Type:     schema.TypeString,
 			Optional: true,
-			ValidateFunc: validation.StringMatch(
-				regexp.MustCompile(`^[0-9a-zA-Z/_-]+$`),
-				"must contain only alphanumeric, hyphens, forward slashes, and underscores characters"),
 		},
 		"bucket_name": {
 			Type:     schema.TypeString,

aws/data_aws_unity_catalog_policy_test.go

@@ -65,6 +65,63 @@ func TestDataAwsUnityCatalogPolicy(t *testing.T) {
 	compareJSON(t, j, p)
 }
 
+func TestDataAwsUnityCatalogPolicyFullKms(t *testing.T) {
+	d, err := qa.ResourceFixture{
+		Read:        true,
+		Resource:    DataAwsUnityCatalogPolicy(),
+		NonWritable: true,
+		ID:          ".",
+		HCL: `
+        aws_account_id = "123456789098"
+        bucket_name = "databricks-bucket"
+        role_name = "databricks-role"
+        kms_name = "arn:aws:kms:us-west-2:111122223333:key/databricks-kms"
+        `,
+	}.Apply(t)
+	assert.NoError(t, err)
+	j := d.Get("json").(string)
+	p := `{
+          "Version": "2012-10-17",
+          "Statement": [
+            {
+              "Effect": "Allow",
+              "Action": [
+                "s3:GetObject",
+                "s3:PutObject",
+                "s3:DeleteObject",
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+              ],
+              "Resource": [
+                "arn:aws:s3:::databricks-bucket/*",
+                "arn:aws:s3:::databricks-bucket"
+              ]
+            },
+            {
+              "Effect": "Allow",
+              "Action": [
+                "sts:AssumeRole"
+              ],
+              "Resource": [
+                "arn:aws:iam::123456789098:role/databricks-role"
+              ]
+            },
+            {
+              "Effect": "Allow",
+              "Action": [
+                "kms:Decrypt",
+                "kms:Encrypt",
+                "kms:GenerateDataKey*"
+              ],
+              "Resource": [
+                "arn:aws:kms:us-west-2:111122223333:key/databricks-kms"
+              ]
+            }
+          ]
+        }`
+	compareJSON(t, j, p)
+}
+
 func TestDataAwsUnityCatalogPolicyWithoutKMS(t *testing.T) {
 	d, err := qa.ResourceFixture{
 		Read:        true,

catalog/bindings/bindings.go

@@ -8,7 +8,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 )
 
-func AddCurrentWorkspaceBindings(ctx context.Context, d *schema.ResourceData, w *databricks.WorkspaceClient, securableName string, securableType string) error {
+func AddCurrentWorkspaceBindings(ctx context.Context, d *schema.ResourceData, w *databricks.WorkspaceClient, securableName string, securableType catalog.UpdateBindingsSecurableType) error {
 	if d.Get("isolation_mode") != "ISOLATED" && d.Get("isolation_mode") != "ISOLATION_MODE_ISOLATED" {
 		return nil
 	}

catalog/permissions/permissions.go

@@ -28,7 +28,9 @@ func NewUnityCatalogPermissionsAPI(ctx context.Context, m any) UnityCatalogPermi
 
 func (a UnityCatalogPermissionsAPI) GetPermissions(securable catalog.SecurableType, name string) (list *catalog.PermissionsList, err error) {
 	if securable.String() == "share" {
-		list, err = a.client.Shares.SharePermissions(a.context, sharing.SharePermissionsRequest{name})
+		list, err = a.client.Shares.SharePermissions(a.context, sharing.SharePermissionsRequest{
+			Name: name,
+		})
 		return
 	}
 	list, err = a.client.Grants.GetBySecurableTypeAndFullName(a.context, securable, name)

catalog/resource_catalog.go

@@ -100,7 +100,7 @@ func ResourceCatalog() common.Resource {
 			}
 
 			// Bind the current workspace if the catalog is isolated, otherwise the read will fail
-			return bindings.AddCurrentWorkspaceBindings(ctx, d, w, ci.Name, "catalog")
+			return bindings.AddCurrentWorkspaceBindings(ctx, d, w, ci.Name, catalog.UpdateBindingsSecurableTypeCatalog)
 		},
 		Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
 			w, err := c.WorkspaceClient()
@@ -166,7 +166,7 @@ func ResourceCatalog() common.Resource {
 			d.SetId(ci.Name)
 
 			// Bind the current workspace if the catalog is isolated, otherwise the read will fail
-			return bindings.AddCurrentWorkspaceBindings(ctx, d, w, ci.Name, "catalog")
+			return bindings.AddCurrentWorkspaceBindings(ctx, d, w, ci.Name, catalog.UpdateBindingsSecurableTypeCatalog)
 		},
 		Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
 			w, err := c.WorkspaceClient()

catalog/resource_external_location.go

@@ -75,7 +75,7 @@ func ResourceExternalLocation() common.Resource {
 			}
 
 			// Bind the current workspace if the external location is isolated, otherwise the read will fail
-			return bindings.AddCurrentWorkspaceBindings(ctx, d, w, el.Name, "external-location")
+			return bindings.AddCurrentWorkspaceBindings(ctx, d, w, el.Name, catalog.UpdateBindingsSecurableTypeExternalLocation)
 		},
 		Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
 			w, err := c.WorkspaceClient()
@@ -134,7 +134,7 @@ func ResourceExternalLocation() common.Resource {
 				return err
 			}
 			// Bind the current workspace if the external location is isolated, otherwise the read will fail
-			return bindings.AddCurrentWorkspaceBindings(ctx, d, w, updateExternalLocationRequest.Name, "external-location")
+			return bindings.AddCurrentWorkspaceBindings(ctx, d, w, updateExternalLocationRequest.Name, catalog.UpdateBindingsSecurableTypeExternalLocation)
 		},
 		Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
 			force := d.Get("force_destroy").(bool)

catalog/resource_external_location_test.go

@@ -92,7 +92,7 @@ func TestCreateIsolatedExternalLocation(t *testing.T) {
 			}, nil)
 			w.GetMockWorkspaceBindingsAPI().EXPECT().UpdateBindings(mock.Anything, catalog.UpdateWorkspaceBindingsParameters{
 				SecurableName: "abc",
-				SecurableType: "external-location",
+				SecurableType: "external_location",
 				Add: []catalog.WorkspaceBinding{
 					{
 						WorkspaceId: int64(123456789101112),

catalog/resource_metastore.go

@@ -67,6 +67,10 @@ func ResourceMetastore() common.Resource {
 			common.DataToStructPointer(d, s, &create)
 			common.DataToStructPointer(d, s, &update)
 			updateForceSendFields(&update)
+			emptyRequest, err := common.IsRequestEmpty(update)
+			if err != nil {
+				return err
+			}
 			return c.AccountOrWorkspaceRequest(func(acc *databricks.AccountClient) error {
 				mi, err := acc.Metastores.Create(ctx,
 					catalog.AccountsCreateMetastore{
@@ -75,10 +79,6 @@ func ResourceMetastore() common.Resource {
 				if err != nil {
 					return err
 				}
-				emptyRequest, err := common.IsRequestEmpty(update)
-				if err != nil {
-					return err
-				}
 				d.SetId(mi.MetastoreInfo.MetastoreId)
 				if emptyRequest {
 					return nil
@@ -97,6 +97,9 @@ func ResourceMetastore() common.Resource {
 					return err
 				}
 				d.SetId(mi.MetastoreId)
+				if emptyRequest {
+					return nil
+				}
 				update.Id = mi.MetastoreId
 				_, err = w.Metastores.Update(ctx, update)
 				if err != nil {

catalog/resource_metastore_assignment.go

@@ -76,6 +76,8 @@ func ResourceMetastoreAssignment() common.Resource {
 					return err
 				}
 				d.Set("metastore_id", ma.MetastoreId)
+				d.Set("default_catalog_name", ma.DefaultCatalogName)
+				d.Set("workspace_id", workspaceId)
 				return nil
 			})
 		},