Merge branch 'main' of github.com:databricks/terraform-provider-datab…

…ricks into migrate-cluster-plugin-framework

clusters/resource_cluster.go

@@ -270,14 +270,31 @@ func FixInstancePoolChangeIfAny(d *schema.ResourceData, cluster any) error {
 func SetForceSendFieldsForCluster(cluster any, d *schema.ResourceData) error {
 	switch c := cluster.(type) {
 	case *compute.ClusterSpec:
+		// Used in jobs.
 		if c.Autoscale == nil {
 			c.ForceSendFields = append(c.ForceSendFields, "NumWorkers")
 		}
+		// Workload type is not relevant in jobs clusters.
 		return nil
 	case *compute.CreateCluster:
 		if c.Autoscale == nil {
 			c.ForceSendFields = append(c.ForceSendFields, "NumWorkers")
 		}
+		// If workload type is set by the user, the fields within Clients should always be sent.
+		// These default to true if not set.
+		if c.WorkloadType != nil {
+			c.WorkloadType.Clients.ForceSendFields = []string{"Jobs", "Notebooks"}
+		}
+		return nil
+	case *compute.EditCluster:
+		if c.Autoscale == nil {
+			c.ForceSendFields = append(c.ForceSendFields, "NumWorkers")
+		}
+		// If workload type is set by the user, the fields within Clients should always be sent.
+		// These default to true if not set.
+		if c.WorkloadType != nil {
+			c.WorkloadType.Clients.ForceSendFields = []string{"Jobs", "Notebooks"}
+		}
 		return nil
 	default:
 		return fmt.Errorf(unsupportedExceptCreateEditClusterSpecErr, cluster, "*", "*", "*")
@@ -309,6 +326,17 @@ func (ClusterSpec) CustomizeSchemaResourceSpecific(s *common.CustomizableSchema)
 			return old == new
 		},
 	})
+	s.AddNewField("no_wait", &schema.Schema{
+		Type:     schema.TypeBool,
+		Optional: true,
+		Default:  false,
+		DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
+			if old == "" && new == "false" {
+				return true
+			}
+			return old == new
+		},
+	})
 	s.AddNewField("state", &schema.Schema{
 		Type:     schema.TypeString,
 		Computed: true,
@@ -414,11 +442,8 @@ func resourceClusterCreate(ctx context.Context, d *schema.ResourceData, c *commo
 	if err != nil {
 		return err
 	}
-	clusterInfo, err := clusterWaiter.GetWithTimeout(timeout)
-	if err != nil {
-		return err
-	}
-	d.SetId(clusterInfo.ClusterId)
+
+	d.SetId(clusterWaiter.ClusterId)
 	d.Set("cluster_id", d.Id())
 	isPinned, ok := d.GetOk("is_pinned")
 	if ok && isPinned.(bool) {
@@ -437,6 +462,20 @@ func resourceClusterCreate(ctx context.Context, d *schema.ResourceData, c *commo
 		}); err != nil {
 			return err
 		}
+	}
+
+	// If there is a no_wait flag set to true, don't wait for the cluster to be created
+	noWait, ok := d.GetOk("no_wait")
+	if ok && noWait.(bool) {
+		return nil
+	}
+
+	clusterInfo, err := clusterWaiter.GetWithTimeout(timeout)
+	if err != nil {
+		return err
+	}
+
+	if len(cluster.Libraries) > 0 {
 		_, err := libraries.WaitForLibrariesInstalledSdk(ctx, w, compute.Wait{
 			ClusterID: d.Id(),
 			IsRunning: clusterInfo.IsRunningOrResizing(),
@@ -508,7 +547,7 @@ func resourceClusterRead(ctx context.Context, d *schema.ResourceData, c *common.
 func hasClusterConfigChanged(d *schema.ResourceData) bool {
 	for k := range clusterSchema {
 		// TODO: create a map if we'll add more non-cluster config parameters in the future
-		if k == "library" || k == "is_pinned" {
+		if k == "library" || k == "is_pinned" || k == "no_wait" {
 			continue
 		}
 		if d.HasChange(k) {
@@ -551,6 +590,7 @@ func resourceClusterUpdate(ctx context.Context, d *schema.ResourceData, c *commo
 		for k := range clusterSchema {
 			if k == "library" ||
 				k == "is_pinned" ||
+				k == "no_wait" ||
 				k == "num_workers" ||
 				k == "autoscale" {
 				continue
@@ -603,10 +643,7 @@ func resourceClusterUpdate(ctx context.Context, d *schema.ResourceData, c *commo
 				Autoscale: cluster.Autoscale,
 			})
 		} else {
-			if err != nil {
-				return err
-			}
-			cluster.ForceSendFields = []string{"NumWorkers"}
+			SetForceSendFieldsForCluster(&cluster, d)
 
 			err = retry.RetryContext(ctx, 15*time.Minute, func() *retry.RetryError {
 				_, err = clusters.Edit(ctx, cluster)

clusters/resource_cluster_test.go

@@ -432,6 +432,161 @@ func TestResourceClusterCreatePhoton(t *testing.T) {
 	assert.Equal(t, "abc", d.Id())
 }
 
+func TestResourceClusterCreateNoWait_WithLibraries(t *testing.T) {
+	d, err := qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "POST",
+				Resource: "/api/2.1/clusters/create",
+				ExpectedRequest: compute.ClusterSpec{
+					NumWorkers:             100,
+					SparkVersion:           "7.1-scala12",
+					NodeTypeId:             "i3.xlarge",
+					AutoterminationMinutes: 60,
+				},
+				Response: compute.ClusterDetails{
+					ClusterId: "abc",
+					State:     compute.StateUnknown,
+				},
+			},
+			{
+				Method:       "GET",
+				ReuseRequest: true,
+				Resource:     "/api/2.1/clusters/get?cluster_id=abc",
+				Response: compute.ClusterDetails{
+					ClusterId:              "abc",
+					NumWorkers:             100,
+					SparkVersion:           "7.1-scala12",
+					NodeTypeId:             "i3.xlarge",
+					AutoterminationMinutes: 15,
+					State:                  compute.StateUnknown,
+				},
+			},
+			{
+				Method:   "POST",
+				Resource: "/api/2.1/clusters/events",
+				ExpectedRequest: compute.GetEvents{
+					ClusterId:  "abc",
+					Limit:      1,
+					Order:      compute.GetEventsOrderDesc,
+					EventTypes: []compute.EventType{compute.EventTypePinned, compute.EventTypeUnpinned},
+				},
+				Response: compute.GetEventsResponse{
+					Events:     []compute.ClusterEvent{},
+					TotalCount: 0,
+				},
+			},
+			{
+				Method:   "POST",
+				Resource: "/api/2.0/libraries/install",
+				ExpectedRequest: compute.InstallLibraries{
+					ClusterId: "abc",
+					Libraries: []compute.Library{
+						{
+							Pypi: &compute.PythonPyPiLibrary{
+								Package: "seaborn==1.2.4",
+							},
+						},
+					},
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/libraries/cluster-status?cluster_id=abc",
+				Response: compute.ClusterLibraryStatuses{
+					LibraryStatuses: []compute.LibraryFullStatus{
+						{
+							Library: &compute.Library{
+								Pypi: &compute.PythonPyPiLibrary{
+									Package: "seaborn==1.2.4",
+								},
+							},
+							Status: compute.LibraryInstallStatusPending,
+						},
+					},
+				},
+			},
+		},
+		Create:   true,
+		Resource: ResourceCluster(),
+		HCL: `num_workers = 100
+		spark_version = "7.1-scala12"
+		node_type_id = "i3.xlarge"
+		no_wait = true
+
+		library {
+			pypi {
+				package = "seaborn==1.2.4"
+			}
+		}`,
+	}.Apply(t)
+	assert.NoError(t, err)
+	assert.Equal(t, "abc", d.Id())
+}
+
+func TestResourceClusterCreateNoWait(t *testing.T) {
+	d, err := qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "POST",
+				Resource: "/api/2.1/clusters/create",
+				ExpectedRequest: compute.ClusterSpec{
+					NumWorkers:             100,
+					ClusterName:            "Shared Autoscaling",
+					SparkVersion:           "7.1-scala12",
+					NodeTypeId:             "i3.xlarge",
+					AutoterminationMinutes: 15,
+				},
+				Response: compute.ClusterDetails{
+					ClusterId: "abc",
+					State:     compute.StateUnknown,
+				},
+			},
+			{
+				Method:       "GET",
+				ReuseRequest: true,
+				Resource:     "/api/2.1/clusters/get?cluster_id=abc",
+				Response: compute.ClusterDetails{
+					ClusterId:              "abc",
+					NumWorkers:             100,
+					ClusterName:            "Shared Autoscaling",
+					SparkVersion:           "7.1-scala12",
+					NodeTypeId:             "i3.xlarge",
+					AutoterminationMinutes: 15,
+					State:                  compute.StateUnknown,
+				},
+			},
+			{
+				Method:   "POST",
+				Resource: "/api/2.1/clusters/events",
+				ExpectedRequest: compute.GetEvents{
+					ClusterId:  "abc",
+					Limit:      1,
+					Order:      compute.GetEventsOrderDesc,
+					EventTypes: []compute.EventType{compute.EventTypePinned, compute.EventTypeUnpinned},
+				},
+				Response: compute.GetEventsResponse{
+					Events:     []compute.ClusterEvent{},
+					TotalCount: 0,
+				},
+			},
+		},
+		Create:   true,
+		Resource: ResourceCluster(),
+		State: map[string]any{
+			"autotermination_minutes": 15,
+			"cluster_name":            "Shared Autoscaling",
+			"spark_version":           "7.1-scala12",
+			"node_type_id":            "i3.xlarge",
+			"num_workers":             100,
+			"is_pinned":               false,
+			"no_wait":                 true,
+		},
+	}.Apply(t)
+	assert.NoError(t, err)
+	assert.Equal(t, "abc", d.Id())
+}
+
 func TestResourceClusterCreate_Error(t *testing.T) {
 	d, err := qa.ResourceFixture{
 		Fixtures: []qa.HTTPFixture{

common/resource.go

@@ -381,30 +381,31 @@ func genericDatabricksData[T, P, C any](
 	var dummy T
 	var other P
 	otherFields := StructToSchema(other, nil)
-	s := StructToSchema(dummy, func(m map[string]*schema.Schema) map[string]*schema.Schema {
-		// For WorkspaceData and AccountData, a single data type is used to represent all of the fields of
-		// the resource, so its configuration is correct. For the *WithParams methods, the SdkType parameter
-		// is copied directly from the resource definition, which means that all fields from that type are
-		// computed and optional, and the fields from OtherFields are overlaid on top of the schema generated
-		// by SdkType.
-		if hasOther {
-			for k := range m {
-				m[k].Computed = true
-				m[k].Required = false
-				m[k].Optional = true
-			}
-			for k, v := range otherFields {
-				m[k] = v
-			}
+
+	s := StructToSchema(dummy, nil)
+	// For WorkspaceData and AccountData, a single data type is used to represent all of the fields of
+	// the resource, so its configuration is correct. For the *WithParams methods, the SdkType parameter
+	// is copied directly from the resource definition, which means that all fields from that type are
+	// computed and optional, and the fields from OtherFields are overlaid on top of the schema generated
+	// by SdkType.
+	if hasOther {
+		for k := range s {
+			s[k].Computed = true
+			s[k].Required = false
+			s[k].Optional = true
 		}
-		// `id` attribute must be marked as computed, otherwise it's not set!
-		if v, ok := m["id"]; ok {
-			v.Computed = true
-			v.Required = false
+		for k, v := range otherFields {
+			s[k] = v
 		}
-		// allow c
-		return customizeSchemaFunc(m)
-	})
+	}
+	// `id` attribute must be marked as computed, otherwise it's not set!
+	if v, ok := s["id"]; ok {
+		v.Computed = true
+		v.Required = false
+	}
+	// allow c
+	s = customizeSchemaFunc(s)
+
 	return Resource{
 		Schema: s,
 		Read: func(ctx context.Context, d *schema.ResourceData, client *DatabricksClient) (err error) {

docs/resources/automatic_cluster_update_setting.md

@@ -33,17 +33,16 @@ resource "databricks_automatic_cluster_update_workspace_setting" "this" {
 
 The resource supports the following arguments:
 
-* `enabled` - (Required) The configuration details.
-* `restart_even_if_no_updates_available` - (Optional) To force clusters and other compute resources to restart during the maintenance window regardless of the availability of a new update.
-
-A `maintenance_window` block that defines the maintenance frequency with the following arguments
-
-* A `week_day_based_schedule` block with the following arguments
-  * `day_of_week` - the day of the week in uppercase, e.g. `MONDAY` or `SUNDAY`
-  * `frequency` - one of the `FIRST_OF_MONTH`, `SECOND_OF_MONTH`, `THIRD_OF_MONTH`, `FOURTH_OF_MONTH`, `FIRST_AND_THIRD_OF_MONTH`, `SECOND_AND_FOURTH_OF_MONTH`, `EVERY_WEEK`.
-  * A `window_start_time` block that defines the time of your maintenance window. The default timezone is UTC and cannot be changed.
-    * `hours`
-    * `minutes`
+- `automatic_cluster_update_workspace` (Required) block with following attributes
+  - `enabled` - (Required) The configuration details.
+  - `restart_even_if_no_updates_available` - (Optional) To force clusters and other compute resources to restart during the maintenance window regardless of the availability of a new update.
+  - `maintenance_window` block that defines the maintenance frequency with the following arguments
+    - `week_day_based_schedule` block with the following arguments
+      - `day_of_week` - the day of the week in uppercase, e.g. `MONDAY` or `SUNDAY`
+      - `frequency` - one of the `FIRST_OF_MONTH`, `SECOND_OF_MONTH`, `THIRD_OF_MONTH`, `FOURTH_OF_MONTH`, `FIRST_AND_THIRD_OF_MONTH`, `SECOND_AND_FOURTH_OF_MONTH`, `EVERY_WEEK`.
+      - `window_start_time` block that defines the time of your maintenance window. The default timezone is UTC and cannot be changed.
+        - `hours` - hour to perform update: 0-23
+        - `minutes` - minute to perform update: 0-59
 
 ## Import
 

docs/resources/cluster.md

@@ -51,6 +51,7 @@ resource "databricks_cluster" "shared_autoscaling" {
 * `custom_tags` - (Optional) Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS EC2 instances and EBS volumes) with these tags in addition to `default_tags`. If a custom cluster tag has the same name as a default cluster tag, the custom tag is prefixed with an `x_` when it is propagated.
 * `spark_conf` - (Optional) Map with key-value pairs to fine-tune Spark clusters, where you can provide custom [Spark configuration properties](https://spark.apache.org/docs/latest/configuration.html) in a cluster configuration.
 * `is_pinned` - (Optional) boolean value specifying if the cluster is pinned (not pinned by default). You must be a Databricks administrator to use this.  The pinned clusters' maximum number is [limited to 100](https://docs.databricks.com/clusters/clusters-manage.html#pin-a-cluster), so `apply` may fail if you have more than that (this number may change over time, so check Databricks documentation for actual number).
+* `no_wait` - (Optional) If true, the provider will not wait for the cluster to reach `RUNNING` state when creating the cluster, allowing cluster creation and library installation to continue asynchronously. Defaults to false (the provider will wait for cluster creation and library installation to succeed).
 
 The following example demonstrates how to create an autoscaling cluster with [Delta Cache](https://docs.databricks.com/delta/optimizations/delta-cache.html) enabled:
 

docs/resources/compliance_security_profile_setting.md

@@ -27,8 +27,9 @@ resource "databricks_compliance_security_profile_workspace_setting" "this" {
 
 The resource supports the following arguments:
 
-* `is_enabled` - (Required) Enable the Compliance Security Profile on the workspace
-* `compliance_standards` - (Required) Enable one or more compliance standards on the workspace, e.g. `HIPAA`, `PCI_DSS`, `FEDRAMP_MODERATE`
+- `compliance_security_profile_workspace` block with following attributes:
+  - `is_enabled` - (Required) Enable the Compliance Security Profile on the workspace
+  - `compliance_standards` - (Required) Enable one or more compliance standards on the workspace, e.g. `HIPAA`, `PCI_DSS`, `FEDRAMP_MODERATE`
 
 ## Import
 

docs/resources/enhanced_security_monitoring_setting.md

@@ -25,7 +25,8 @@ resource "databricks_enhanced_security_monitoring_workspace_setting" "this" {
 
 The resource supports the following arguments:
 
-* `is_enabled` - (Required) Enable the Enhanced Security Monitoring on the workspace
+ - `enhanced_security_monitoring_workspace` block with following attributes:
+   - `is_enabled` - (Required) Enable the Enhanced Security Monitoring on the workspace
 
 ## Import
 

docs/resources/job.md

@@ -231,7 +231,7 @@ One of the `query`, `dashboard` or `alert` needs to be provided.
   * `pause_subscriptions` - (Optional) flag that specifies if subscriptions are paused or not.
 * `alert` - (Optional) block consisting of following fields:
   * `alert_id` - (Required) (String) identifier of the Databricks SQL Alert.
-  * `subscriptions` - (Required) a list of subscription blocks consisting out of one of the required fields: `user_name` for user emails or `destination_id` - for Alert destination's identifier.
+  * `subscriptions` - (Optional) a list of subscription blocks consisting out of one of the required fields: `user_name` for user emails or `destination_id` - for Alert destination's identifier.
   * `pause_subscriptions` - (Optional) flag that specifies if subscriptions are paused or not.
 * `file` - (Optional) block consisting of single string fields:
   * `source` - (Optional) The source of the project. Possible values are `WORKSPACE` and `GIT`.