Merge branch 'main' into aastha/upgrade-google-module

CHANGELOG.md

@@ -2,6 +2,63 @@
 
 All notable changes to this project will be documented in this file.
 
+### [5.0.1](https://github.com/wandb/terraform-google-wandb/compare/v5.0.0...v5.0.1) (2024-07-22)
+
+
+### Bug Fixes
+
+* Weave SA ([#152](https://github.com/wandb/terraform-google-wandb/issues/152)) ([81aca11](https://github.com/wandb/terraform-google-wandb/commit/81aca117e939d93cc4769ae3106fda706cb62f60))
+
+## [5.0.0](https://github.com/wandb/terraform-google-wandb/compare/v4.0.3...v5.0.0) (2024-07-18)
+
+
+### ⚠ BREAKING CHANGES
+
+* Service Account Mapping (#151)
+
+### Features
+
+* Service Account Mapping ([#151](https://github.com/wandb/terraform-google-wandb/issues/151)) ([8930eaf](https://github.com/wandb/terraform-google-wandb/commit/8930eafde09a7013d57e2dec045685765d2d84c4))
+
+### [4.0.3](https://github.com/wandb/terraform-google-wandb/compare/v4.0.2...v4.0.3) (2024-07-18)
+
+
+### Bug Fixes
+
+* Kms sa name ([#150](https://github.com/wandb/terraform-google-wandb/issues/150)) ([1a70cdf](https://github.com/wandb/terraform-google-wandb/commit/1a70cdff42ef3b35386386b674af5643e23fbd00))
+
+### [4.0.2](https://github.com/wandb/terraform-google-wandb/compare/v4.0.1...v4.0.2) (2024-07-17)
+
+
+### Bug Fixes
+
+* Stackdriver SA name ([#148](https://github.com/wandb/terraform-google-wandb/issues/148)) ([e67c9fc](https://github.com/wandb/terraform-google-wandb/commit/e67c9fc5a09459893ebc7960e9c40a315cc9e6f3))
+
+### [4.0.1](https://github.com/wandb/terraform-google-wandb/compare/v4.0.0...v4.0.1) (2024-07-16)
+
+
+### Bug Fixes
+
+* Stackdriver SA regex ([#147](https://github.com/wandb/terraform-google-wandb/issues/147)) ([aa9dfc5](https://github.com/wandb/terraform-google-wandb/commit/aa9dfc52c95f8355e216f4dab5b3f428d17931cf))
+
+## [4.0.0](https://github.com/wandb/terraform-google-wandb/compare/v3.7.0...v4.0.0) (2024-07-15)
+
+
+### ⚠ BREAKING CHANGES
+
+* Index error and missing breaking change (#146)
+
+### Bug Fixes
+
+* Index error and missing breaking change ([#146](https://github.com/wandb/terraform-google-wandb/issues/146)) ([3e2c484](https://github.com/wandb/terraform-google-wandb/commit/3e2c48477117cb39687e68d08d5b06c7d595cbde))
+
+## [3.7.0](https://github.com/wandb/terraform-google-wandb/compare/v3.6.1...v3.7.0) (2024-07-15)
+
+
+### Features
+
+* Added namespace as a prefix in stackdriver sa name ([#144](https://github.com/wandb/terraform-google-wandb/issues/144)) ([af49f8b](https://github.com/wandb/terraform-google-wandb/commit/af49f8b2afe67f14d0b9f4648a7133985cc4626d))
+
 ### [3.6.1](https://github.com/wandb/terraform-google-wandb/compare/v3.6.0...v3.6.1) (2024-07-11)
 
 

README.md

@@ -119,7 +119,6 @@ resources that lack official modules.
 | <a name="input_gke_machine_type"></a> [gke\_machine\_type](#input\_gke\_machine\_type) | Specifies the machine type to be allocated for the database | `string` | `"n1-standard-4"` | no |
 | <a name="input_gke_node_count"></a> [gke\_node\_count](#input\_gke\_node\_count) | n/a | `number` | `2` | no |
 | <a name="input_ilb_proxynetwork_cidr"></a> [ilb\_proxynetwork\_cidr](#input\_ilb\_proxynetwork\_cidr) | Internal load balancer proxy subnetwork | `string` | `"10.127.0.0/24"` | no |
-| <a name="input_kms_gcs_sa_name"></a> [kms\_gcs\_sa\_name](#input\_kms\_gcs\_sa\_name) | n/a | `string` | `"wandb-app"` | no |
 | <a name="input_labels"></a> [labels](#input\_labels) | Labels to apply to resources | `map(string)` | `{}` | no |
 | <a name="input_license"></a> [license](#input\_license) | Your wandb/local license | `string` | n/a | yes |
 | <a name="input_local_restore"></a> [local\_restore](#input\_local\_restore) | Restores W&B to a stable state if needed | `bool` | `false` | no |
@@ -173,3 +172,10 @@ resources that lack official modules.
 | <a name="output_standardized_size"></a> [standardized\_size](#output\_standardized\_size) | n/a |
 | <a name="output_url"></a> [url](#output\_url) | The URL to the W&B application |
 <!-- END_TF_DOCS -->
+
+## Migrations
+
+### 3.x -> 4.x 
+
+3.6.0 introduced a change in the Google Provider that isn't backwards compatible with prior versions. 
+Nothing needs to be done to upgrade, but it is not backwards compatible. 

main.tf

@@ -26,13 +26,20 @@ locals {
   url            = "${local.url_prefix}://${local.fqdn}"
   create_bucket  = var.bucket_name == ""
   create_network = var.network == null
+  k8s_sa_map = {
+    app         = "wandb-app"
+    parquet     = "wandb-parquet"
+    flat_runs   = "wandb-flat-run-fields-updater"
+    weave       = "wandb-weave"
+    weave_trace = "wandb-weave-trace"
+  }
 }
 
 module "service_accounts" {
   source                   = "./modules/service_accounts"
   namespace                = var.namespace
   bucket_name              = var.bucket_name
-  kms_gcs_sa_name          = var.kms_gcs_sa_name
+  kms_gcs_sa_list          = values(local.k8s_sa_map)
   create_workload_identity = var.create_workload_identity
   stackdriver_sa_name      = var.stackdriver_sa_name
   enable_stackdriver       = var.enable_stackdriver
@@ -199,6 +206,10 @@ locals {
   internal_lb_name = "${var.namespace}-internal"
 }
 
+locals {
+  workload_hash = var.create_workload_identity ? substr(sha256("yes"), 0, 50) : null
+}
+
 data "google_client_config" "current" {}
 
 module "wandb" {
@@ -208,8 +219,9 @@ module "wandb" {
   spec = {
     values = {
       global = {
-        host    = local.url
-        license = var.license
+        pod           = { labels = { workload_hash : local.workload_hash } }
+        host          = local.url
+        license       = var.license
         cloudProvider = "gcp"
         extraEnv = merge({
           "GORILLA_DISABLE_CODE_SAVING"          = var.disable_code_saving,
@@ -257,8 +269,8 @@ module "wandb" {
       app = {
         extraEnvs = var.app_wandb_env
         serviceAccount = var.create_workload_identity ? {
-          name        = var.kms_gcs_sa_name
-          annotations = { "iam.gke.io/gcp-service-account" = module.service_accounts.sa_account_email }
+          name        = local.k8s_sa_map.app
+          annotations = { "iam.gke.io/gcp-service-account" = module.service_accounts.sa_account_role }
           } : {
           name        = ""
           annotations = {}
@@ -291,7 +303,7 @@ module "wandb" {
           projectId          = data.google_client_config.current.project
           serviceAccountName = var.stackdriver_sa_name
         }
-        serviceAccount = { annotations = { "iam.gke.io/gcp-service-account" = module.service_accounts.stackdriver_email } }
+        serviceAccount = { annotations = { "iam.gke.io/gcp-service-account" = module.service_accounts.stackdriver_role } }
         } : {
         install        = false
         stackdriver    = {}
@@ -339,10 +351,44 @@ module "wandb" {
 
       weave = {
         extraEnvs = var.weave_wandb_env
+        serviceAccount = var.create_workload_identity ? {
+          name        = local.k8s_sa_map.weave
+          annotations = { "iam.gke.io/gcp-service-account" = module.service_accounts.sa_account_role }
+          } : {
+          name        = null
+          annotations = {}
+        }
+      }
+
+      weave-trace = {
+        serviceAccount = var.create_workload_identity ? {
+          name        = local.k8s_sa_map.weave_trace
+          annotations = { "iam.gke.io/gcp-service-account" = module.service_accounts.sa_account_role }
+          } : {
+          name        = null
+          annotations = {}
+        }
       }
 
       parquet = {
         extraEnvs = var.parquet_wandb_env
+        serviceAccount = var.create_workload_identity ? {
+          name        = local.k8s_sa_map.parquet
+          annotations = { "iam.gke.io/gcp-service-account" = module.service_accounts.sa_account_role }
+          } : {
+          name        = null
+          annotations = {}
+        }
+      }
+
+      flat-runs-fields-updater = {
+        serviceAccount = var.create_workload_identity ? {
+          name        = local.k8s_sa_map.flat_runs
+          annotations = { "iam.gke.io/gcp-service-account" = module.service_accounts.sa_account_role }
+          } : {
+          name        = null
+          annotations = {}
+        }
       }
     }
   }
@@ -413,4 +459,4 @@ module "private_link" {
   psc_subnetwork        = var.psc_subnetwork_cidr
   proxynetwork_cidr     = var.ilb_proxynetwork_cidr
   depends_on            = [google_compute_subnetwork.proxy, data.google_compute_forwarding_rules.all]
-}
+}

modules/app_gke/main.tf

@@ -77,14 +77,9 @@ resource "google_container_node_pool" "default" {
       "https://www.googleapis.com/auth/trace.append",
       "https://www.googleapis.com/auth/sqlservice.admin",
     ]
-
-     dynamic "workload_metadata_config" {
-       for_each = var.create_workload_identity == true ? [1] : []
-        content {
-          mode = "GKE_METADATA"
-        }
-      }
-
+    workload_metadata_config {
+      mode = var.create_workload_identity ? "GKE_METADATA" : "GCE_METADATA"
+    }
     shielded_instance_config {
       enable_secure_boot = true
     }

modules/service_accounts/main.tf

@@ -62,10 +62,11 @@ resource "google_project_iam_member" "secretmanager_admin" {
   role    = "roles/secretmanager.admin"
 }
 
-####### service account for kms and gcs cross project access
+
+####### service account for kms and gcs
 resource "google_service_account" "kms_gcs_sa" {
   count        = var.create_workload_identity == true ? 1 : 0
-  account_id   = var.kms_gcs_sa_name
+  account_id   = format("%s-kms-gcs", substr(random_id.main.dec, 0, 22))
   display_name = "Service Account For Workload Identity"
 }
 
@@ -83,8 +84,6 @@ resource "google_project_iam_member" "secretmanager_admin_gcs" {
   role    = "roles/secretmanager.admin"
 }
 
-# For some reason we need this permission otherwise backend is throwing an error
-# hopfully this is a short term fix.
 resource "google_project_iam_member" "log_writer_gcs" {
   count   = var.create_workload_identity == true ? 1 : 0
   project = local.project_id
@@ -100,13 +99,12 @@ resource "google_project_iam_member" "storage" {
 }
 
 resource "google_storage_bucket_iam_member" "gcs_admin" {
-  count  = var.bucket_name != "" ? 1 : 0
+  count  = var.create_workload_identity == true && var.bucket_name != "" ? 1 : 0
   bucket = var.bucket_name
   member = google_service_account.kms_gcs_sa[0].email
   role   = "roles/storage.objectAdmin"
 }
 
-
 resource "google_project_iam_member" "kms" {
   count   = var.create_workload_identity == true ? 1 : 0
   project = local.project_id
@@ -115,44 +113,43 @@ resource "google_project_iam_member" "kms" {
 }
 
 resource "google_service_account_iam_member" "token_creator_binding" {
-  count   = var.create_workload_identity == true ? 1 : 0
+  count              = var.create_workload_identity == true ? 1 : 0
   service_account_id = google_service_account.kms_gcs_sa[0].id
-  role    = "roles/iam.serviceAccountTokenCreator"
-  member = "serviceAccount:${google_service_account.kms_gcs_sa[0].email}"
+  role               = "roles/iam.serviceAccountTokenCreator"
+  member             = "serviceAccount:${google_service_account.kms_gcs_sa[0].email}"
 }
 
 resource "google_service_account_iam_member" "workload_binding" {
-  count   = var.create_workload_identity == true ? 1 : 0
+  for_each           = var.create_workload_identity ? { for sa in var.kms_gcs_sa_list : sa => sa } : {}
   service_account_id = google_service_account.kms_gcs_sa[0].id
-  role    = "roles/iam.workloadIdentityUser"
-  member  = "serviceAccount:${local.project_id}.svc.id.goog[default/${var.kms_gcs_sa_name}]"
+  role               = "roles/iam.workloadIdentityUser"
+  member             = "serviceAccount:${local.project_id}.svc.id.goog[default/${each.value}]"
 }
 
-
 ### service account for stackdriver
 resource "google_service_account" "stackdriver" {
   count        = var.enable_stackdriver == true ? 1 : 0
-  account_id   = var.stackdriver_sa_name
+  account_id   = format("%s-stackdriver", substr(random_id.main.dec, 0, 18))
   display_name = "Service Account For Workload Identity"
 }
 
 resource "google_project_iam_member" "monitoring" {
   count   = var.enable_stackdriver == true ? 1 : 0
   project = local.project_id
   role    = "roles/monitoring.viewer"
-  member = "serviceAccount:${google_service_account.stackdriver[0].email}"
+  member  = "serviceAccount:${google_service_account.stackdriver[0].email}"
 }
 
 resource "google_service_account_iam_member" "stackdriver_token_creator" {
-  count   = var.enable_stackdriver == true ? 1 : 0
+  count              = var.enable_stackdriver == true ? 1 : 0
   service_account_id = google_service_account.stackdriver[0].id
-  role    = "roles/iam.serviceAccountTokenCreator"
-  member = "serviceAccount:${google_service_account.stackdriver[0].email}"
+  role               = "roles/iam.serviceAccountTokenCreator"
+  member             = "serviceAccount:${google_service_account.stackdriver[0].email}"
 }
 
 resource "google_service_account_iam_member" "stackdriver_binding" {
-  count   = var.enable_stackdriver == true ? 1 : 0
+  count              = var.enable_stackdriver == true ? 1 : 0
   service_account_id = google_service_account.stackdriver[0].id
-  role    = "roles/iam.workloadIdentityUser"
-  member  = "serviceAccount:${local.project_id}.svc.id.goog[default/${var.stackdriver_sa_name}]"
-}
+  role               = "roles/iam.workloadIdentityUser"
+  member             = "serviceAccount:${local.project_id}.svc.id.goog[default/${var.stackdriver_sa_name}]"
+}

modules/service_accounts/outputs.tf

@@ -3,10 +3,10 @@ output "service_account" {
   description = "The service account."
 }
 
-output "sa_account_email" {
+output "sa_account_role" {
   value = var.create_workload_identity == true ? google_service_account.kms_gcs_sa[0].email : null
 }
 
-output "stackdriver_email" {
+output "stackdriver_role" {
   value = var.enable_stackdriver == true ? google_service_account.stackdriver[0].email : null
 }

modules/service_accounts/variables.tf

@@ -9,14 +9,13 @@ variable "bucket_name" {
   default     = ""
 }
 
-
 variable "create_workload_identity" {
   description = "Flag to indicate whether to create a workload identity for the service account."
   type        = bool
 }
 
-variable "kms_gcs_sa_name" {
-  type    = string
+variable "kms_gcs_sa_list" {
+  type = list(string)
 }
 
 variable "stackdriver_sa_name" {

outputs.tf

@@ -92,5 +92,5 @@ output "private_attachement_id" {
 
 output "sa_account_email" {
   description =  "This output provides the email address of the service account created for workload identity, if workload identity is enabled. Otherwise, it returns null"
-  value = var.create_workload_identity == true ? module.service_accounts.sa_account_email : null
+  value = var.create_workload_identity == true ? module.service_accounts.sa_account_role : null
 }

variables.tf

@@ -269,7 +269,7 @@ variable "create_private_link" {
 variable "public_access" {
   type        = bool
   description = "Whether to create a public endpoint for wandb access."
-  default = true
+  default     = true
 }
 
 variable "allowed_project_names" {
@@ -298,18 +298,13 @@ variable "create_workload_identity" {
   default     = false
 }
 
-variable "kms_gcs_sa_name" {
-  type    = string
-  default = "wandb-app"
-}
 
 variable "enable_stackdriver" {
-  type = bool
+  type    = bool
   default = false
 }
 
 variable "stackdriver_sa_name" {
   type    = string
   default = "wandb-stackdriver"
 }
-