Merge pull request #104 from databricks/gcp

Gcp update - Optional Segregated Network operations

gcp/modules/workspace_deployment/cmek.tf

@@ -20,16 +20,17 @@ resource "google_kms_crypto_key" "databricks_key" {
 
 
 
-
 # # assign CMEK on Databricks side
 resource "databricks_mws_customer_managed_keys" "this" {
+
         provider = databricks.accounts
         account_id   = var.databricks_account_id
         gcp_key_info {
-            kms_key_id   = google_kms_crypto_key.databricks_key[0].id 
+            kms_key_id   = var.use_existing_cmek? "projects/${var.google_project}/locations/${var.google_region}/keyRings/${var.keyring_name}/cryptoKeys/${var.key_name}": google_kms_crypto_key.databricks_key[0].id
         }
         use_cases = ["STORAGE","MANAGED","MANAGED_SERVICES"]
         lifecycle {
               ignore_changes = all
         }
-}
+}
+

gcp/modules/workspace_deployment/init.tf

@@ -2,14 +2,11 @@ variable "databricks_account_id" {}
 variable "databricks_google_service_account" {}
 variable "google_project" {}
 variable "google_region" {}
+# variable "google_zone" {}
 
 variable "workspace_pe" {}
 variable "relay_pe" {}
 
-variable "use_existing_cmek" {}
-variable "key_name" {}
-variable "keyring_name" {}
-variable "cmek_resource_id" {}
 
 variable "account_console_url" {}
 
@@ -20,6 +17,15 @@ variable "google_pe_subnet" {}
 variable "relay_pe_ip_name" {}
 variable "workspace_pe_ip_name" {}
 
+# For the value of the regional Hive Metastore IP, refer to the Databricks documentation
+# Here - https://docs.gcp.databricks.com/en/resources/ip-domain-region.html
+variable "hive_metastore_ip" {}
+
+variable "use_existing_cmek" {}
+variable "key_name" {}
+variable "keyring_name" {}
+
+
 variable "google_pe_subnet_ip_cidr_range" {
   default = "10.3.0.0/24"
 }
@@ -43,16 +49,48 @@ variable "mws_workspace_gke_master_ip_range" {
   default = "10.3.0.0/28"
 }
 
-//Users can connect to workspace only from this list of IP's
+variable "use_existing_vpc" {
+  default = false
+}
+variable "existing_vpc_name" {
+  default = ""
+}
+variable "existing_subnet_name" {
+  default = ""
+}
+variable "existing_pod_range_name"{
+  default = ""
+}
+variable "existing_service_range_name"{
+  default = ""
+}
+
+variable "use_existing_PSC_EP" {
+  default = false
+}
+
+
+variable "harden_network" {
+  # Flag to enable Firewall setup by the current module
+  default = true
+}
+
+
+//Users can connect to workspace only thes list of IP's
 variable "ip_addresses" {
   type = list(string)
 }
 
-// Regional value for the Hive Metastore IP (allowing Egress to this public IP)
-variable "hive_metastore_ip" {
-  default = "34.76.244.202"
+variable "cmek_resource_id" {
+  default = ""
+}
+variable "use_existing_pas" {}
+variable "existing_pas_id" {
+  default = ""
+}
+variable "workspace_name" {
+  default = "tf-demo-test"
 }
-
 
 
 
@@ -72,14 +110,17 @@ terraform {
     }
     google = {
       source  = "hashicorp/google"
-      version = ">= 6.2.0"
+      version = ">=5.43.1"
+
     }
   }
 }
 
 provider "google" {
   project = var.google_project
   region  = var.google_region
+  impersonate_service_account = var.databricks_google_service_account
+  # zone    = var.google_zone
 }
 
 // initialize provider in "accounts" mode to provision new workspace
@@ -123,7 +164,8 @@ resource "databricks_user" "me" {
 
 
  provider  = databricks.workspace
- user_name = data.google_client_openid_userinfo.me.email
+#  user_name = data.google_client_openid_userinfo.me.email
+ user_name = "aleksander.callebat@databricks.com"
 }
 
 

gcp/modules/workspace_deployment/psc.tf

@@ -1,41 +1,46 @@
 resource "google_compute_subnetwork" "backend_pe_subnetwork" {
+  count = var.use_existing_PSC_EP ? 0 : 1
   name          = var.google_pe_subnet
   ip_cidr_range = var.google_pe_subnet_ip_cidr_range
   region        = var.google_region
   network       = google_compute_network.dbx_private_vpc.id
   secondary_ip_range {
-    range_name    = "tf-test-secondary-range-update1"
+    range_name    = "tf-secondary-range-update1"
     ip_cidr_range = var.google_pe_subnet_secondary_ip_range
   }
-    private_ip_google_access = true
+  private_ip_google_access = true
 
-            depends_on=[google_compute_network.dbx_private_vpc]
+  depends_on=[google_compute_network.dbx_private_vpc]
 }
 
 
 resource "google_compute_forwarding_rule" "backend_psc_ep" {
+  count = var.use_existing_PSC_EP ? 0 : 1
   depends_on = [
     google_compute_address.backend_pe_ip_address, google_compute_network.dbx_private_vpc
   ]
   region      = var.google_region
   project     = var.google_project
   name        = var.relay_pe
   network     = google_compute_network.dbx_private_vpc.id
-  ip_address  = google_compute_address.backend_pe_ip_address.id
+  ip_address  = google_compute_address.backend_pe_ip_address[0].id
   target      = var.relay_service_attachment
   load_balancing_scheme = "" #This field must be set to "" if the target is an URI of a service attachment. Default value is EXTERNAL
 }
 
 resource "google_compute_address" "backend_pe_ip_address" {
+  count = var.use_existing_PSC_EP ? 0 : 1
   name         = var.relay_pe_ip_name
   provider     = google
   project      = var.google_project
   region       = var.google_region
-  subnetwork   = google_compute_subnetwork.backend_pe_subnetwork.name
+  subnetwork   = google_compute_subnetwork.backend_pe_subnetwork[0].name
   address_type = "INTERNAL"
 }
 
 resource "google_compute_forwarding_rule" "frontend_psc_ep" {
+  count = var.use_existing_PSC_EP ? 0 : 1
+
   depends_on = [
     google_compute_address.frontend_pe_ip_address
   ]
@@ -44,24 +49,51 @@ resource "google_compute_forwarding_rule" "frontend_psc_ep" {
   project     = var.google_project
   network     = google_compute_network.dbx_private_vpc.id
 
-  ip_address  = google_compute_address.frontend_pe_ip_address.id
+  ip_address  = google_compute_address.frontend_pe_ip_address[0].id
   target      = var.workspace_service_attachment
   load_balancing_scheme = "" #This field must be set to "" if the target is an URI of a service attachment. Default value is EXTERNAL
 }
 
 resource "google_compute_address" "frontend_pe_ip_address" {
+  count = var.use_existing_PSC_EP ? 0 : 1
+
   name         = var.workspace_pe_ip_name
   provider     = google
   project      = var.google_project
   region       = var.google_region
-  subnetwork   = google_compute_subnetwork.backend_pe_subnetwork.name
+  subnetwork   = google_compute_subnetwork.backend_pe_subnetwork[0].name
   address_type = "INTERNAL"
 }
 
+resource "databricks_mws_vpc_endpoint" "backend_rest_vpce" {
+  depends_on =[google_compute_forwarding_rule.backend_psc_ep]
+  provider     = databricks.accounts
+  account_id          = var.databricks_account_id
+  vpc_endpoint_name   = "vpce-backend-rest"
+  gcp_vpc_endpoint_info {
+   project_id        = var.google_project
+   psc_endpoint_name = var.workspace_pe
+   endpoint_region   = google_compute_subnetwork.network-with-private-secondary-ip-ranges.region
+ }
+}
+
+resource "databricks_mws_vpc_endpoint" "relay_vpce" {
+  provider     = databricks.accounts
+  depends_on = [ google_compute_forwarding_rule.frontend_psc_ep ]
+  account_id          = var.databricks_account_id
+  vpc_endpoint_name   = "vpce-relay"
+  gcp_vpc_endpoint_info {
+    project_id        = var.google_project
+    psc_endpoint_name = var.relay_pe
+    endpoint_region   = google_compute_subnetwork.network-with-private-secondary-ip-ranges.region
+ }
+}
+
 output "front_end_psc_status"{
-  value = "Frontend psc status: ${google_compute_forwarding_rule.frontend_psc_ep.psc_connection_status}"
+  value = "Frontend psc status: ${var.use_existing_PSC_EP?"Pre-provisioned":google_compute_forwarding_rule.frontend_psc_ep[0].psc_connection_status}"
 }
 
 output "backend_end_psc_status"{
-  value = "Backend psc status: ${google_compute_forwarding_rule.backend_psc_ep.psc_connection_status}"
-}
+  value = "Backend psc status: ${var.use_existing_PSC_EP?"Pre-provisioned":google_compute_forwarding_rule.backend_psc_ep[0].psc_connection_status}"
+}
+

gcp/modules/workspace_deployment/vpc-firewall.tf

@@ -1,4 +1,5 @@
 resource "google_compute_firewall" "deny_egress" {
+  count = var.harden_network ? 1 : 0
   name                    = "deny-egress-${google_compute_network.dbx_private_vpc.name}"
   direction               = "EGRESS"
   priority                = 1100
@@ -15,6 +16,7 @@ resource "google_compute_firewall" "deny_egress" {
 # This is the only Egress rule that goes to a public internet IP
 # It can be avoided if the workspace is UC-enabled and that the spark config is configured to avoid fetching the metastore IP
 resource "google_compute_firewall" "to_databricks_managed_hive" {
+  count = var.harden_network ? 1 : 0
   name                    = "to-databricks-managed-hive-${google_compute_network.dbx_private_vpc.name}"
   direction               = "EGRESS"
   priority                = 1010
@@ -28,6 +30,7 @@ resource "google_compute_firewall" "to_databricks_managed_hive" {
 }
 
 resource "google_compute_firewall" "to_gke_health_checks" {
+  count = var.harden_network ? 1 : 0
   name                    = "to-gke-health-checks-${google_compute_network.dbx_private_vpc.name}"
   direction               = "EGRESS"
   priority                = 1010
@@ -41,6 +44,7 @@ resource "google_compute_firewall" "to_gke_health_checks" {
 }
 
 resource "google_compute_firewall" "from_gke_health_checks" {
+  count = var.harden_network ? 1 : 0
   name                    = "from-gke-health-checks-${google_compute_network.dbx_private_vpc.name}"
   direction               = "INGRESS"
   priority                = 1010
@@ -54,6 +58,7 @@ resource "google_compute_firewall" "from_gke_health_checks" {
 }
 
 resource "google_compute_firewall" "to_gke_cp" {
+  count = var.harden_network ? 1 : 0
   name                    = "to-gke-cp-${google_compute_network.dbx_private_vpc.name}"
   direction               = "EGRESS"
   priority                = 1010
@@ -67,6 +72,7 @@ resource "google_compute_firewall" "to_gke_cp" {
 }
 
 resource "google_compute_firewall" "to_google_apis" {
+  count = var.harden_network ? 1 : 0
   name                    = "to-google-apis-${google_compute_network.dbx_private_vpc.name}"
   direction               = "EGRESS"
   priority                = 1010
@@ -79,6 +85,7 @@ resource "google_compute_firewall" "to_google_apis" {
 }
 
 resource "google_compute_firewall" "to_gke_nodes_subnet" {
+  count = var.harden_network ? 1 : 0
   name                    = "to-gke-nodes-subnet-${google_compute_network.dbx_private_vpc.name}"
   direction               = "EGRESS"
   priority                = 1010

gcp/modules/workspace_deployment/vpc.tf

@@ -5,7 +5,7 @@ resource "google_compute_network" "dbx_private_vpc" {
 }
 
 resource "google_compute_subnetwork" "network-with-private-secondary-ip-ranges" {
-  name          = "test-dbx-${random_string.suffix.result}"
+  name          = "ws-subnet-dbx-${random_string.suffix.result}"
   ip_cidr_range = var.nodes_ip_cidr_range
   region        = var.google_region
   network       = google_compute_network.dbx_private_vpc.id
@@ -20,58 +20,17 @@ resource "google_compute_subnetwork" "network-with-private-secondary-ip-ranges"
   private_ip_google_access = true
 }
 
-resource "google_compute_router" "router" {
-  name    = "my-router-${random_string.suffix.result}"
-  region  = google_compute_subnetwork.network-with-private-secondary-ip-ranges.region
-  network = google_compute_network.dbx_private_vpc.id
-}
-
-resource "google_compute_router_nat" "nat" {
-  name                               = "my-router-nat-${random_string.suffix.result}"
-  router                             = google_compute_router.router.name
-  region                             = google_compute_router.router.region
-  nat_ip_allocate_option             = "AUTO_ONLY"
-  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
-}
-
-resource "databricks_mws_vpc_endpoint" "backend_rest_vpce" {
-depends_on =[google_compute_forwarding_rule.backend_psc_ep]
- provider     = databricks.accounts
-
- account_id          = var.databricks_account_id
- vpc_endpoint_name   = "vpce-backend-rest-${random_string.suffix.result}"
- gcp_vpc_endpoint_info {
-   project_id        = var.google_project
-   psc_endpoint_name = var.workspace_pe
-   endpoint_region   = google_compute_subnetwork.network-with-private-secondary-ip-ranges.region
- }
-}
-
-resource "databricks_mws_vpc_endpoint" "relay_vpce" {
-  depends_on = [ google_compute_forwarding_rule.frontend_psc_ep ]
- provider     = databricks.accounts
-
- account_id          = var.databricks_account_id
- vpc_endpoint_name   = "vpce-relay-${random_string.suffix.result}"
- gcp_vpc_endpoint_info {
-   project_id        = var.google_project
-   psc_endpoint_name = var.relay_pe
-   endpoint_region   = google_compute_subnetwork.network-with-private-secondary-ip-ranges.region
- }
-}
-
-
-resource "databricks_mws_networks" "this" {
+resource "databricks_mws_networks" "network_config" {
   provider     = databricks.accounts
   account_id   = var.databricks_account_id
-  network_name = "test-demo-${random_string.suffix.result}"
+  network_name = "config-eu1-${random_string.suffix.result}"
   gcp_network_info {
     network_project_id    = var.google_project
-    vpc_id                = google_compute_network.dbx_private_vpc.name
-    subnet_id             = google_compute_subnetwork.network-with-private-secondary-ip-ranges.name
-    subnet_region         = google_compute_subnetwork.network-with-private-secondary-ip-ranges.region
-    pod_ip_range_name     = "pods"
-    service_ip_range_name = "svc"
+    vpc_id                = var.use_existing_vpc? var.existing_vpc_name:google_compute_network.dbx_private_vpc.name
+    subnet_id             = var.use_existing_vpc?var.existing_subnet_name:google_compute_subnetwork.network-with-private-secondary-ip-ranges.name
+    subnet_region         = var.google_region
+    pod_ip_range_name     = var.use_existing_vpc?var.existing_pod_range_name:"pods"
+    service_ip_range_name = var.use_existing_vpc?var.existing_service_range_name:"svc"
   }
   vpc_endpoints {