From 8dde5cf198f3435d3560f6ad8c70b9e6462508ea Mon Sep 17 00:00:00 2001 From: velotioaastha Date: Wed, 8 May 2024 17:57:53 +0530 Subject: [PATCH] add example tf files for custom vpc, sql, redis --- examples/byo-vpc-eks-sql-redis/main.tf | 339 ++++++++++++++ examples/byo-vpc-eks-sql-redis/variables.tf | 490 ++++++++++++++++++++ examples/byo-vpc-sql/main.tf | 384 +++++++++++++++ examples/byo-vpc-sql/variables.tf | 463 ++++++++++++++++++ 4 files changed, 1676 insertions(+) create mode 100644 examples/byo-vpc-eks-sql-redis/main.tf create mode 100644 examples/byo-vpc-eks-sql-redis/variables.tf create mode 100644 examples/byo-vpc-sql/main.tf create mode 100644 examples/byo-vpc-sql/variables.tf diff --git a/examples/byo-vpc-eks-sql-redis/main.tf b/examples/byo-vpc-eks-sql-redis/main.tf new file mode 100644 index 000000000..9f4bf70f9 --- /dev/null +++ b/examples/byo-vpc-eks-sql-redis/main.tf @@ -0,0 +1,339 @@ +provider "aws" { + region = "us-east-1" + + + default_tags { + tags = { + GithubRepo = "terraform-aws-wandb" + GithubOrg = "wandb" + Enviroment = "Example" + Example = "BYO-VPC-EKS-SQL-REDIS" + } + } +} +data "aws_s3_bucket" "file_storage" { + depends_on = [module.file_storage] + bucket = local.bucket_name +} + +data "aws_sqs_queue" "file_storage" { + count = local.use_internal_queue ? 0 : 1 + depends_on = [module.file_storage] + name = local.bucket_queue_name +} + +data "aws_eks_cluster" "app_cluster" { + name = var.eks_cluster_name +} + +data "aws_eks_cluster_auth" "app_cluster" { + name = var.eks_cluster_name +} + +provider "kubernetes" { + host = data.aws_eks_cluster.app_cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.app_cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.app_cluster.token + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.app_cluster.name] + command = "aws" + } +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.app_cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.app_cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.app_cluster.token + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.app_cluster.name] + command = "aws" + } + } +} + +module "kms" { + source = "../../modules/kms" + + key_alias = var.kms_key_alias == null ? "${var.namespace}-kms-alias" : var.kms_key_alias + key_deletion_window = var.kms_key_deletion_window + + key_policy = var.kms_key_policy +} + +locals { + kms_key_arn = module.kms.key.arn + use_external_bucket = var.bucket_name != "" + use_internal_queue = local.use_external_bucket || var.use_internal_queue + deployment_size = { + small = { + db = "db.r6g.large", + node_count = 3, + node_instance = "r6i.xlarge" + cache = "cache.m6g.large" + }, + medium = { + db = "db.r6g.xlarge", + node_count = 3, + node_instance = "r6i.xlarge" + cache = "cache.m6g.large" + }, + large = { + db = "db.r6g.2xlarge", + node_count = 3, + node_instance = "r6i.2xlarge" + cache = "cache.m6g.xlarge" + }, + xlarge = { + db = "db.r6g.4xlarge", + node_count = 3, + node_instance = "r6i.2xlarge" + cache = "cache.m6g.xlarge" + }, + xxlarge = { + db = "db.r6g.8xlarge", + node_count = 3, + node_instance = "r6i.4xlarge" + cache = "cache.m6g.2xlarge" + } + } +} + +module "file_storage" { + count = var.create_bucket ? 1 : 0 + source = "../../modules/file_storage" + + create_queue = !local.use_internal_queue + deletion_protection = var.deletion_protection + kms_key_arn = local.kms_key_arn + namespace = var.namespace + sse_algorithm = "aws:kms" +} + +locals { + bucket_name = local.use_external_bucket ? var.bucket_name : module.file_storage.0.bucket_name + bucket_queue_name = local.use_internal_queue ? null : module.file_storage.0.bucket_queue_name +} + +module "networking" { + source = "../../modules/networking" + namespace = var.namespace + create_vpc = var.create_vpc + + cidr = var.network_cidr + private_subnet_cidrs = var.network_private_subnet_cidrs + public_subnet_cidrs = var.network_public_subnet_cidrs + database_subnet_cidrs = var.network_database_subnet_cidrs + create_elasticache_subnet = var.create_elasticache + elasticache_subnet_cidrs = var.network_elasticache_subnet_cidrs +} + +locals { + network_id = var.create_vpc ? module.networking.vpc_id : var.network_id + network_public_subnets = var.create_vpc ? module.networking.public_subnets : var.network_public_subnets + + network_private_subnets = var.create_vpc ? module.networking.private_subnets : var.network_private_subnets + network_private_subnet_cidrs = var.create_vpc ? module.networking.private_subnet_cidrs : var.network_private_subnet_cidrs + + network_database_subnets = var.create_vpc ? module.networking.database_subnets : var.network_database_subnets + network_database_subnet_cidrs = var.create_vpc ? module.networking.database_subnet_cidrs : var.network_database_subnet_cidrs + network_database_create_subnet_group = !var.create_vpc + network_database_subnet_group_name = var.create_vpc ? module.networking.database_subnet_group_name : "${var.namespace}-database-subnet" +} + +locals { + create_certificate = var.public_access && var.acm_certificate_arn == null + + fqdn = var.subdomain == null ? var.domain_name : "${var.subdomain}.${var.domain_name}" +} + +# Create SSL Ceritifcation if applicable +module "acm" { + source = "terraform-aws-modules/acm/aws" + version = "~> 3.0" + + create_certificate = local.create_certificate + + subject_alternative_names = var.extra_fqdn + + domain_name = var.external_dns ? local.fqdn : var.domain_name + zone_id = var.zone_id + + wait_for_validation = true +} + +locals { + acm_certificate_arn = local.create_certificate ? module.acm.acm_certificate_arn : var.acm_certificate_arn + url = local.acm_certificate_arn == null ? "http://${local.fqdn}" : "https://${local.fqdn}" + domain_filter = var.custom_domain_filter == null || var.custom_domain_filter == "" ? local.fqdn : var.custom_domain_filter + + internal_app_port = 32543 +} + +locals { + full_fqdn = var.enable_dummy_dns ? "old.${local.fqdn}" : local.fqdn + extra_fqdn = var.enable_dummy_dns ? [for fqdn in var.extra_fqdn : "old.${fqdn}"] : var.extra_fqdn +} + +module "app_lb" { + source = "../../modules/app_lb" + + namespace = var.namespace + load_balancing_scheme = var.public_access ? "PUBLIC" : "PRIVATE" + acm_certificate_arn = local.acm_certificate_arn + zone_id = var.zone_id + + fqdn = local.full_fqdn + extra_fqdn = local.extra_fqdn + allowed_inbound_cidr = var.allowed_inbound_cidr + allowed_inbound_ipv6_cidr = var.allowed_inbound_ipv6_cidr + target_port = local.internal_app_port + + network_id = local.network_id + network_private_subnets = local.network_private_subnets + network_public_subnets = local.network_public_subnets +} + +module "private_link" { + count = length(var.private_link_allowed_account_ids) > 0 ? 1 : 0 + source = "../../modules/private_link" + + namespace = var.namespace + allowed_account_ids = var.private_link_allowed_account_ids + deletion_protection = var.deletion_protection + network_private_subnets = local.network_private_subnets + alb_name = local.lb_name_truncated + vpc_id = local.network_id + + depends_on = [ + module.wandb + ] +} + +resource "aws_autoscaling_attachment" "autoscaling_attachment" { + for_each = var.autoscaling_group_names + autoscaling_group_name = each.value + lb_target_group_arn = module.app_lb.tg_app_arn +} + +locals { + network_elasticache_subnets = var.create_vpc ? module.networking.elasticache_subnets : var.network_elasticache_subnets + network_elasticache_subnet_cidrs = var.create_vpc ? module.networking.elasticache_subnet_cidrs : var.network_elasticache_subnet_cidrs + network_elasticache_create_subnet_group = !var.create_vpc + network_elasticache_subnet_group_name = var.create_vpc ? module.networking.elasticache_subnet_group_name : "${var.namespace}-elasticache-subnet" +} + +module "redis" { + count = var.create_elasticache ? 1 : 0 + redis_create_subnet_group = local.network_elasticache_create_subnet_group + redis_subnets = local.network_elasticache_subnets + source = "../../modules/redis" + namespace = var.namespace + + vpc_id = local.network_id + redis_subnet_group_name = local.network_elasticache_subnet_group_name + vpc_subnets_cidr_blocks = local.network_elasticache_subnet_cidrs + node_type = try(local.deployment_size[var.size].cache, var.elasticache_node_type) + kms_key_arn = local.kms_key_arn +} + +locals { + max_lb_name_length = 32 - length("-alb-k8s") + lb_name_truncated = "${substr(var.namespace, 0, local.max_lb_name_length)}-alb-k8s" +} + +module "wandb" { + source = "wandb/wandb/helm" + version = "1.2.0" + + operator_chart_version = "1.1.2" + controller_image_tag = "1.10.1" + + spec = { + values = { + global = { + host = local.url + license = var.license + + extraEnv = var.other_wandb_env + + bucket = { + provider = "s3" + name = local.bucket_name + region = data.aws_s3_bucket.file_storage.region + kmsKey = local.use_external_bucket ? var.bucket_kms_key_arn : local.kms_key_arn + } + + mysql = { + host = var.database_endpoint + password = var.database_master_password + user = var.database_master_username + database = var.database_name + port = var.database_port + } + + redis = { + host = var.create_elasticache ? module.redis.0.host : var.redis_host + port = var.create_elasticache ? "${module.redis.0.port}?tls=true&ttlInSeconds=604800" : "${var.redis_port}?tls=true&ttlInSeconds=604800" + } + } + + ingress = { + class = "alb" + + additionalHosts = concat(var.extra_fqdn, length(var.private_link_allowed_account_ids) > 0 ? [""] : []) + + annotations = merge({ + "alb.ingress.kubernetes.io/load-balancer-name" = local.lb_name_truncated + "alb.ingress.kubernetes.io/inbound-cidrs" = <<-EOF + ${join("\\,", var.allowed_inbound_cidr)} + EOF + "external-dns.alpha.kubernetes.io/ingress-hostname-source" = "annotation-only" + "alb.ingress.kubernetes.io/scheme" = var.kubernetes_alb_internet_facing ? "internet-facing" : "internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/listen-ports" = "[{\\\"HTTPS\\\": 443}]" + "alb.ingress.kubernetes.io/certificate-arn" = local.acm_certificate_arn + }, + length(var.extra_fqdn) > 0 && var.enable_dummy_dns ? { + "external-dns.alpha.kubernetes.io/hostname" = <<-EOF + ${local.fqdn}\,${join("\\,", var.extra_fqdn)}\,${local.fqdn} + EOF + } : { + "external-dns.alpha.kubernetes.io/hostname" = var.enable_operator_alb ? local.fqdn : "" + }, + length(var.kubernetes_alb_subnets) > 0 ? { + "alb.ingress.kubernetes.io/subnets" = <<-EOF + ${join("\\,", var.kubernetes_alb_subnets)} + EOF + } : {}) + + } + + app = var.enable_operator_alb ? {} : { + extraEnv = merge({ + "GORILLA_GLUE_LIST" = "true" + }, var.app_wandb_env) + } + + mysql = { install = false } + redis = { install = false } + + weave = { + persistence = { + provider = "efs" + efs = { + fileSystemId = var.efs_id + } + } + extraEnv = var.weave_wandb_env + } + + parquet = { + extraEnv = var.parquet_wandb_env + } + } + } +} + diff --git a/examples/byo-vpc-eks-sql-redis/variables.tf b/examples/byo-vpc-eks-sql-redis/variables.tf new file mode 100644 index 000000000..0743e96dd --- /dev/null +++ b/examples/byo-vpc-eks-sql-redis/variables.tf @@ -0,0 +1,490 @@ +########################################## +# Common # +########################################## +variable "namespace" { + type = string + description = "String used for prefix resources." +} + +variable "deletion_protection" { + description = "If the instance should have deletion protection enabled. The database / S3 can't be deleted when this value is set to `true`." + type = bool + default = false +} + +variable "use_internal_queue" { + type = bool + default = false +} + +variable "size" { + default = null + description = "Deployment size" + nullable = true + type = string +} + +########################################## +# Database # +########################################## +variable "database_engine_version" { + description = "Version for MySQL Auora" + type = string + default = "8.0.mysql_aurora.3.05.2" +} + +variable "database_instance_class" { + description = "Instance type to use by database master instance." + type = string + default = "db.r5.large" +} + +variable "database_snapshot_identifier" { + description = "Specifies whether or not to create this cluster from a snapshot. You can use either the name or ARN when specifying a DB cluster snapshot, or the ARN when specifying a DB snapshot" + type = string + default = null +} + +variable "database_sort_buffer_size" { + description = "Specifies the sort_buffer_size value to set for the database" + type = number + default = 67108864 +} + +variable "database_name" { + description = "Specifies the name of the database" + type = string +} + +variable "database_port" { + description = "Specifies the port of the database" + type = string +} + +variable "database_master_username" { + description = "Specifies the master_username value to set for the database" + type = string +} + +variable "database_master_password" { + description = "Specifies the master_password value to set for the database" + type = string + sensitive = true +} + +variable "database_endpoint" { + description = "Specifies the endpoint value to set for the database" + type = string +} + +variable "database_binlog_format" { + description = "Specifies the binlog_format value to set for the database" + type = string + default = "ROW" +} + +variable "database_innodb_lru_scan_depth" { + description = "Specifies the innodb_lru_scan_depth value to set for the database" + type = number + default = 128 +} + +variable "database_performance_insights_kms_key_arn" { + default = null + description = "Specifies an existing KMS key ARN to encrypt the performance insights data if performance_insights_enabled is was enabled out of band" + nullable = true + type = string +} + +variable "database_security_group_id" { + description = "Specifies the security group id value to set for the database" + type = string +} + +########################################## +# DNS # +########################################## +variable "public_access" { + type = bool + default = false + description = "Is this instance accessable a public domain." +} + +variable "external_dns" { + type = bool + default = false + description = "Using external DNS. A `subdomain` must also be specified if this value is true." +} + +variable "custom_domain_filter" { + description = "A custom domain filter to be used by external-dns instead of the default FQDN. If not set, the local FQDN is used." + type = string + default = null +} + +# Sometimes domain name and zone name dont match, so lets explicitly ask for +# both. Also is just life easier to have both even though in most cause it may +# be redundant info. +# https://github.com/hashicorp/terraform-aws-terraform-enterprise/pull/41#issuecomment-563501858 +variable "zone_id" { + type = string + description = "Domain for creating the Weights & Biases subdomain on." +} + +variable "domain_name" { + type = string + description = "Domain for accessing the Weights & Biases UI." +} + +variable "subdomain" { + type = string + default = null + description = "Subdomain for accessing the Weights & Biases UI. Default creates record at Route53 Route." +} + +variable "enable_dummy_dns" { + type = bool + default = false + description = "Boolean indicating whether or not to enable dummy DNS for the old alb" +} + + +variable "enable_operator_alb" { + type = bool + default = false + description = "Boolean indicating whether to use operatore ALB (true) or not (false)." +} + +variable "extra_fqdn" { + type = list(string) + description = "Additional fqdn's must be in the same hosted zone as `domain_name`." + default = [] +} + +########################################## +# Load Balancer # +########################################## +variable "ssl_policy" { + type = string + default = "ELBSecurityPolicy-FS-1-2-Res-2020-10" + description = "SSL policy to use on ALB listener" +} + +variable "acm_certificate_arn" { + type = string + default = null + description = "The ARN of an existing ACM certificate." +} + +variable "allowed_inbound_cidr" { + description = "CIDRs allowed to access wandb-server." + nullable = false + type = list(string) +} + +variable "allowed_inbound_ipv6_cidr" { + description = "CIDRs allowed to access wandb-server." + nullable = false + type = list(string) +} + + +########################################## +# KMS # +########################################## +variable "kms_key_alias" { + type = string + description = "KMS key alias for AWS KMS Customer managed key." + default = null +} + +variable "kms_key_deletion_window" { + type = number + description = "Duration in days to destroy the key after it is deleted. Must be between 7 and 30 days." + default = 7 +} + +variable "kms_key_policy" { + type = string + description = "The policy that will define the permissions for the kms key." + default = "" +} + +########################################## +# Network # +########################################## +variable "create_vpc" { + type = bool + description = "Boolean indicating whether to deploy a VPC (true) or not (false)." + default = false +} + +variable "network_id" { + default = "" + description = "The identity of the VPC in which resources will be deployed." + type = string +} + +variable "network_private_subnets" { + default = [] + description = "A list of the identities of the private subnetworks in which resources will be deployed." + type = list(string) +} + +variable "network_public_subnets" { + default = [] + description = "A list of the identities of the public subnetworks in which resources will be deployed." + type = list(string) +} + +variable "network_database_subnets" { + default = [] + description = "A list of the identities of the database subnetworks in which resources will be deployed." + type = list(string) +} + +variable "network_elasticache_subnets" { + default = [] + description = "A list of the identities of the subnetworks in which elasticache resources will be deployed." + type = list(string) +} + +variable "network_cidr" { + type = string + description = "CIDR block for VPC." + default = "10.10.0.0/16" +} + +variable "network_public_subnet_cidrs" { + type = list(string) + description = "List of private subnet CIDR ranges to create in VPC." + default = ["10.10.0.0/24", "10.10.1.0/24"] +} + +variable "network_private_subnet_cidrs" { + type = list(string) + description = "List of private subnet CIDR ranges to create in VPC." + default = ["10.10.10.0/24", "10.10.11.0/24"] +} + +variable "network_database_subnet_cidrs" { + type = list(string) + description = "List of private subnet CIDR ranges to create in VPC." + default = ["10.10.20.0/24", "10.10.21.0/24"] +} + +variable "network_elasticache_subnet_cidrs" { + type = list(string) + description = "List of private subnet CIDR ranges to create in VPC." + default = ["10.10.30.0/24", "10.10.31.0/24"] +} + +variable "private_link_allowed_account_ids" { + description = "List of AWS account IDs allowed to access the VPC Endpoint Service" + type = list(string) + default = [] +} + +########################################## +# EKS Cluster # +########################################## +variable "eks_cluster_name" { + description = "EKS cluster kubernetes name" + nullable = false + type = string +} + +variable "eks_cluster_version" { + description = "EKS cluster kubernetes version" + nullable = false + type = string +} + +variable "efs_id" { + description = "EFS id" + type = string +} + +variable "autoscaling_group_names" { + type = map(string) + default = { + "primary": "" + } +} + +variable "kubernetes_alb_internet_facing" { + type = bool + description = "Indicates whether or not the ALB controlled by the Amazon ALB ingress controller is internet-facing or internal." + default = true +} + +variable "kubernetes_alb_subnets" { + type = list(string) + description = "List of subnet ID's the ALB will use for ingress traffic." + default = [] +} + +variable "kubernetes_public_access" { + type = bool + description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled." + default = true +} + + +variable "kubernetes_public_access_cidrs" { + description = "List of CIDR blocks which can access the Amazon EKS public API server endpoint." + type = list(string) + default = [] +} + +variable "kubernetes_map_accounts" { + description = "Additional AWS account numbers to add to the aws-auth configmap." + type = list(string) + default = [] +} + +variable "kubernetes_map_roles" { + description = "Additional IAM roles to add to the aws-auth configmap." + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) + default = [] +} + +variable "kubernetes_map_users" { + description = "Additional IAM users to add to the aws-auth configmap." + type = list(object({ + userarn = string + username = string + groups = list(string) + })) + default = [] +} + +variable "kubernetes_instance_types" { + description = "EC2 Instance type for primary node group." + type = list(string) + default = ["m5.large"] +} + +variable "kubernetes_node_count" { + description = "Number of nodes" + type = number + default = 2 +} + +variable "eks_policy_arns" { + type = list(string) + description = "Additional IAM policy to apply to the EKS cluster" + default = [] +} + +variable "system_reserved_cpu_millicores" { + description = "(Optional) The amount of 'system-reserved' CPU millicores to pass to the kubelet. For example: 100. A value of -1 disables the flag." + type = number + default = 70 +} + +variable "system_reserved_memory_megabytes" { + description = "(Optional) The amount of 'system-reserved' memory in megabytes to pass to the kubelet. For example: 100. A value of -1 disables the flag." + type = number + default = 100 +} + +variable "system_reserved_ephemeral_megabytes" { + description = "(Optional) The amount of 'system-reserved' ephemeral storage in megabytes to pass to the kubelet. For example: 1000. A value of -1 disables the flag." + type = number + default = 750 +} + +variable "system_reserved_pid" { + description = "(Optional) The amount of 'system-reserved' process ids [pid] to pass to the kubelet. For example: 1000. A value of -1 disables the flag." + type = number + default = 500 +} + +variable "aws_loadbalancer_controller_tags" { + description = "(Optional) A map of AWS tags to apply to all resources managed by the load balancer controller" + type = map(string) + default = {} +} + +########################################## +# External Bucket # +########################################## +# Most users will not need these settings. They are ment for users who want a +# bucket and sqs that are in a different account. +variable "create_bucket" { + type = bool + default = true +} + +variable "bucket_name" { + type = string + default = "" +} + +variable "bucket_kms_key_arn" { + type = string + description = "The Amazon Resource Name of the KMS key with which S3 storage bucket objects will be encrypted." + default = "" +} + +########################################## +# Redis # +########################################## +variable "create_elasticache" { + type = bool + description = "Boolean indicating whether to provision an elasticache instance (true) or not (false)." + default = false +} + +variable "elasticache_node_type" { + description = "The type of the redis cache node to deploy" + type = string + default = "cache.t2.medium" +} + +variable "redis_port" { + description = "Redis port" + type = string +} + +variable "redis_host" { + description = "Redis host" + type = string +} + +# ########################################## +# # Weights & Biases # +# ########################################## +variable "license" { + type = string + description = "Weights & Biases license key." +} + +variable "other_wandb_env" { + type = map(any) + description = "Extra environment variables for W&B" + default = {} +} + +variable "weave_wandb_env" { + type = map(string) + description = "Extra environment variables for W&B" + default = {} +} + +variable "app_wandb_env" { + type = map(string) + description = "Extra environment variables for W&B" + default = {} +} + +variable "parquet_wandb_env" { + type = map(string) + description = "Extra environment variables for W&B" + default = {} +} diff --git a/examples/byo-vpc-sql/main.tf b/examples/byo-vpc-sql/main.tf new file mode 100644 index 000000000..a6d6b1461 --- /dev/null +++ b/examples/byo-vpc-sql/main.tf @@ -0,0 +1,384 @@ +provider "aws" { + region = "us-east-1" + + + default_tags { + tags = { + GithubRepo = "terraform-aws-wandb" + GithubOrg = "wandb" + Enviroment = "Example" + Example = "BYO-VPC-SQL" + } + } +} +data "aws_s3_bucket" "file_storage" { + depends_on = [module.file_storage] + bucket = local.bucket_name +} + +data "aws_sqs_queue" "file_storage" { + count = local.use_internal_queue ? 0 : 1 + depends_on = [module.file_storage] + name = local.bucket_queue_name +} + +data "aws_eks_cluster" "app_cluster" { + name = module.app_eks.cluster_id +} + +data "aws_eks_cluster_auth" "app_cluster" { + name = module.app_eks.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.app_cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.app_cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.app_cluster.token + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.app_cluster.name] + command = "aws" + } +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.app_cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.app_cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.app_cluster.token + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", data.aws_eks_cluster.app_cluster.name] + command = "aws" + } + } +} + +module "kms" { + source = "../../modules/kms" + + key_alias = var.kms_key_alias == null ? "${var.namespace}-kms-alias" : var.kms_key_alias + key_deletion_window = var.kms_key_deletion_window + + key_policy = var.kms_key_policy +} + +locals { + kms_key_arn = module.kms.key.arn + use_external_bucket = var.bucket_name != "" + use_internal_queue = local.use_external_bucket || var.use_internal_queue + deployment_size = { + small = { + db = "db.r6g.large", + node_count = 3, + node_instance = "r6i.xlarge" + cache = "cache.m6g.large" + }, + medium = { + db = "db.r6g.xlarge", + node_count = 3, + node_instance = "r6i.xlarge" + cache = "cache.m6g.large" + }, + large = { + db = "db.r6g.2xlarge", + node_count = 3, + node_instance = "r6i.2xlarge" + cache = "cache.m6g.xlarge" + }, + xlarge = { + db = "db.r6g.4xlarge", + node_count = 3, + node_instance = "r6i.2xlarge" + cache = "cache.m6g.xlarge" + }, + xxlarge = { + db = "db.r6g.8xlarge", + node_count = 3, + node_instance = "r6i.4xlarge" + cache = "cache.m6g.2xlarge" + } + } +} + +module "file_storage" { + count = var.create_bucket ? 1 : 0 + source = "../../modules/file_storage" + + create_queue = !local.use_internal_queue + deletion_protection = var.deletion_protection + kms_key_arn = local.kms_key_arn + namespace = var.namespace + sse_algorithm = "aws:kms" +} + +locals { + bucket_name = local.use_external_bucket ? var.bucket_name : module.file_storage.0.bucket_name + bucket_queue_name = local.use_internal_queue ? null : module.file_storage.0.bucket_queue_name +} + +module "networking" { + source = "../../modules/networking" + namespace = var.namespace + create_vpc = var.create_vpc + + cidr = var.network_cidr + private_subnet_cidrs = var.network_private_subnet_cidrs + public_subnet_cidrs = var.network_public_subnet_cidrs + database_subnet_cidrs = var.network_database_subnet_cidrs + create_elasticache_subnet = var.create_elasticache + elasticache_subnet_cidrs = var.network_elasticache_subnet_cidrs +} + +locals { + network_id = var.create_vpc ? module.networking.vpc_id : var.network_id + network_public_subnets = var.create_vpc ? module.networking.public_subnets : var.network_public_subnets + + network_private_subnets = var.create_vpc ? module.networking.private_subnets : var.network_private_subnets + network_private_subnet_cidrs = var.create_vpc ? module.networking.private_subnet_cidrs : var.network_private_subnet_cidrs + + network_database_subnets = var.create_vpc ? module.networking.database_subnets : var.network_database_subnets + network_database_subnet_cidrs = var.create_vpc ? module.networking.database_subnet_cidrs : var.network_database_subnet_cidrs + network_database_create_subnet_group = !var.create_vpc + network_database_subnet_group_name = var.create_vpc ? module.networking.database_subnet_group_name : "${var.namespace}-database-subnet" +} + +locals { + create_certificate = var.public_access && var.acm_certificate_arn == null + + fqdn = var.subdomain == null ? var.domain_name : "${var.subdomain}.${var.domain_name}" +} + +# Create SSL Ceritifcation if applicable +module "acm" { + source = "terraform-aws-modules/acm/aws" + version = "~> 3.0" + + create_certificate = local.create_certificate + + subject_alternative_names = var.extra_fqdn + + domain_name = var.external_dns ? local.fqdn : var.domain_name + zone_id = var.zone_id + + wait_for_validation = true +} + +locals { + acm_certificate_arn = local.create_certificate ? module.acm.acm_certificate_arn : var.acm_certificate_arn + url = local.acm_certificate_arn == null ? "http://${local.fqdn}" : "https://${local.fqdn}" + domain_filter = var.custom_domain_filter == null || var.custom_domain_filter == "" ? local.fqdn : var.custom_domain_filter + + internal_app_port = 32543 +} + +module "app_eks" { + source = "../../modules/app_eks" + + fqdn = local.domain_filter + + namespace = var.namespace + kms_key_arn = local.kms_key_arn + + instance_types = try([local.deployment_size[var.size].node_instance], var.kubernetes_instance_types) + desired_capacity = try(local.deployment_size[var.size].node_count, var.kubernetes_node_count) + map_accounts = var.kubernetes_map_accounts + map_roles = var.kubernetes_map_roles + map_users = var.kubernetes_map_users + + bucket_kms_key_arn = local.use_external_bucket ? var.bucket_kms_key_arn : local.kms_key_arn + bucket_arn = data.aws_s3_bucket.file_storage.arn + bucket_sqs_queue_arn = local.use_internal_queue ? null : data.aws_sqs_queue.file_storage.0.arn + + network_id = local.network_id + network_private_subnets = local.network_private_subnets + + lb_security_group_inbound_id = module.app_lb.security_group_inbound_id + database_security_group_id = var.database_security_group_id + + create_elasticache_security_group = var.create_elasticache + elasticache_security_group_id = var.create_elasticache ? module.redis.0.security_group_id : null + + cluster_version = var.eks_cluster_version + cluster_endpoint_public_access = var.kubernetes_public_access + cluster_endpoint_public_access_cidrs = var.kubernetes_public_access_cidrs + + eks_policy_arns = var.eks_policy_arns + + system_reserved_cpu_millicores = var.system_reserved_cpu_millicores + system_reserved_memory_megabytes = var.system_reserved_memory_megabytes + system_reserved_ephemeral_megabytes = var.system_reserved_ephemeral_megabytes + system_reserved_pid = var.system_reserved_pid + + aws_loadbalancer_controller_tags = var.aws_loadbalancer_controller_tags +} + +locals { + full_fqdn = var.enable_dummy_dns ? "old.${local.fqdn}" : local.fqdn + extra_fqdn = var.enable_dummy_dns ? [for fqdn in var.extra_fqdn : "old.${fqdn}"] : var.extra_fqdn +} + +module "app_lb" { + source = "../../modules/app_lb" + + namespace = var.namespace + load_balancing_scheme = var.public_access ? "PUBLIC" : "PRIVATE" + acm_certificate_arn = local.acm_certificate_arn + zone_id = var.zone_id + + fqdn = local.full_fqdn + extra_fqdn = local.extra_fqdn + allowed_inbound_cidr = var.allowed_inbound_cidr + allowed_inbound_ipv6_cidr = var.allowed_inbound_ipv6_cidr + target_port = local.internal_app_port + + network_id = local.network_id + network_private_subnets = local.network_private_subnets + network_public_subnets = local.network_public_subnets +} + +module "private_link" { + count = length(var.private_link_allowed_account_ids) > 0 ? 1 : 0 + source = "../../modules/private_link" + + namespace = var.namespace + allowed_account_ids = var.private_link_allowed_account_ids + deletion_protection = var.deletion_protection + network_private_subnets = local.network_private_subnets + alb_name = local.lb_name_truncated + vpc_id = local.network_id + + depends_on = [ + module.wandb + ] +} + +resource "aws_autoscaling_attachment" "autoscaling_attachment" { + for_each = module.app_eks.autoscaling_group_names + autoscaling_group_name = each.value + lb_target_group_arn = module.app_lb.tg_app_arn +} + +locals { + network_elasticache_subnets = var.create_vpc ? module.networking.elasticache_subnets : var.network_elasticache_subnets + network_elasticache_subnet_cidrs = var.create_vpc ? module.networking.elasticache_subnet_cidrs : var.network_elasticache_subnet_cidrs + network_elasticache_create_subnet_group = !var.create_vpc + network_elasticache_subnet_group_name = var.create_vpc ? module.networking.elasticache_subnet_group_name : "${var.namespace}-elasticache-subnet" +} + +module "redis" { + count = var.create_elasticache ? 1 : 0 + redis_create_subnet_group = local.network_elasticache_create_subnet_group + redis_subnets = local.network_elasticache_subnets + source = "../../modules/redis" + namespace = var.namespace + + vpc_id = local.network_id + redis_subnet_group_name = local.network_elasticache_subnet_group_name + vpc_subnets_cidr_blocks = local.network_elasticache_subnet_cidrs + node_type = try(local.deployment_size[var.size].cache, var.elasticache_node_type) + kms_key_arn = local.kms_key_arn +} + +locals { + max_lb_name_length = 32 - length("-alb-k8s") + lb_name_truncated = "${substr(var.namespace, 0, local.max_lb_name_length)}-alb-k8s" +} + +module "wandb" { + source = "wandb/wandb/helm" + version = "1.2.0" + + depends_on = [ + module.app_eks, + module.redis, + ] + operator_chart_version = "1.1.2" + controller_image_tag = "1.10.1" + + spec = { + values = { + global = { + host = local.url + license = var.license + + extraEnv = var.other_wandb_env + + bucket = { + provider = "s3" + name = local.bucket_name + region = data.aws_s3_bucket.file_storage.region + kmsKey = local.use_external_bucket ? var.bucket_kms_key_arn : local.kms_key_arn + } + + mysql = { + host = var.database_endpoint + password = var.database_master_password + user = var.database_master_username + database = var.database_name + port = var.database_port + } + + redis = { + host = module.redis.0.host + port = "${module.redis.0.port}?tls=true&ttlInSeconds=604800" + } + } + + ingress = { + class = "alb" + + additionalHosts = concat(var.extra_fqdn, length(var.private_link_allowed_account_ids) > 0 ? [""] : []) + + annotations = merge({ + "alb.ingress.kubernetes.io/load-balancer-name" = local.lb_name_truncated + "alb.ingress.kubernetes.io/inbound-cidrs" = <<-EOF + ${join("\\,", var.allowed_inbound_cidr)} + EOF + "external-dns.alpha.kubernetes.io/ingress-hostname-source" = "annotation-only" + "alb.ingress.kubernetes.io/scheme" = var.kubernetes_alb_internet_facing ? "internet-facing" : "internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/listen-ports" = "[{\\\"HTTPS\\\": 443}]" + "alb.ingress.kubernetes.io/certificate-arn" = local.acm_certificate_arn + }, + length(var.extra_fqdn) > 0 && var.enable_dummy_dns ? { + "external-dns.alpha.kubernetes.io/hostname" = <<-EOF + ${local.fqdn}\,${join("\\,", var.extra_fqdn)}\,${local.fqdn} + EOF + } : { + "external-dns.alpha.kubernetes.io/hostname" = var.enable_operator_alb ? local.fqdn : "" + }, + length(var.kubernetes_alb_subnets) > 0 ? { + "alb.ingress.kubernetes.io/subnets" = <<-EOF + ${join("\\,", var.kubernetes_alb_subnets)} + EOF + } : {}) + + } + + app = var.enable_operator_alb ? {} : { + extraEnv = merge({ + "GORILLA_GLUE_LIST" = "true" + }, var.app_wandb_env) + } + + mysql = { install = false } + redis = { install = false } + + weave = { + persistence = { + provider = "efs" + efs = { + fileSystemId = module.app_eks.efs_id + } + } + extraEnv = var.weave_wandb_env + } + + parquet = { + extraEnv = var.parquet_wandb_env + } + } + } +} + diff --git a/examples/byo-vpc-sql/variables.tf b/examples/byo-vpc-sql/variables.tf new file mode 100644 index 000000000..47a29b85f --- /dev/null +++ b/examples/byo-vpc-sql/variables.tf @@ -0,0 +1,463 @@ +########################################## +# Common # +########################################## +variable "namespace" { + type = string + description = "String used for prefix resources." +} + +variable "deletion_protection" { + description = "If the instance should have deletion protection enabled. The database / S3 can't be deleted when this value is set to `true`." + type = bool + default = true +} + +variable "use_internal_queue" { + type = bool + default = false +} + +variable "size" { + default = null + description = "Deployment size" + nullable = true + type = string +} + +########################################## +# Database # +########################################## +variable "database_engine_version" { + description = "Version for MySQL Auora" + type = string + default = "8.0.mysql_aurora.3.05.2" +} + +variable "database_instance_class" { + description = "Instance type to use by database master instance." + type = string + default = "db.r5.large" +} + +variable "database_snapshot_identifier" { + description = "Specifies whether or not to create this cluster from a snapshot. You can use either the name or ARN when specifying a DB cluster snapshot, or the ARN when specifying a DB snapshot" + type = string + default = null +} + +variable "database_sort_buffer_size" { + description = "Specifies the sort_buffer_size value to set for the database" + type = number + default = 67108864 +} + +variable "database_name" { + description = "Specifies the name of the database" + type = string + default = "wandb_local" +} + +variable "database_port" { + description = "Specifies the port of the database" + type = string +} + +variable "database_master_username" { + description = "Specifies the master_username value to set for the database" + type = string +} + +variable "database_master_password" { + description = "Specifies the master_password value to set for the database" + type = string + sensitive = true +} + +variable "database_endpoint" { + description = "Specifies the endpoint value to set for the database" + type = string +} + +variable "database_binlog_format" { + description = "Specifies the binlog_format value to set for the database" + type = string + default = "ROW" +} + +variable "database_innodb_lru_scan_depth" { + description = "Specifies the innodb_lru_scan_depth value to set for the database" + type = number + default = 128 +} + +variable "database_performance_insights_kms_key_arn" { + default = null + description = "Specifies an existing KMS key ARN to encrypt the performance insights data if performance_insights_enabled is was enabled out of band" + nullable = true + type = string +} + +variable "database_security_group_id" { + description = "Specifies the security group id value to set for the database" + type = string + default = "sg-08a535514ea4cd7b1" +} + +########################################## +# DNS # +########################################## +variable "public_access" { + type = bool + default = false + description = "Is this instance accessable a public domain." +} + +variable "external_dns" { + type = bool + default = false + description = "Using external DNS. A `subdomain` must also be specified if this value is true." +} + +variable "custom_domain_filter" { + description = "A custom domain filter to be used by external-dns instead of the default FQDN. If not set, the local FQDN is used." + type = string + default = null +} + +# Sometimes domain name and zone name dont match, so lets explicitly ask for +# both. Also is just life easier to have both even though in most cause it may +# be redundant info. +# https://github.com/hashicorp/terraform-aws-terraform-enterprise/pull/41#issuecomment-563501858 +variable "zone_id" { + type = string + description = "Domain for creating the Weights & Biases subdomain on." +} + +variable "domain_name" { + type = string + description = "Domain for accessing the Weights & Biases UI." +} + +variable "subdomain" { + type = string + default = null + description = "Subdomain for accessing the Weights & Biases UI. Default creates record at Route53 Route." +} + +variable "enable_dummy_dns" { + type = bool + default = false + description = "Boolean indicating whether or not to enable dummy DNS for the old alb" +} + + +variable "enable_operator_alb" { + type = bool + default = false + description = "Boolean indicating whether to use operatore ALB (true) or not (false)." +} + +variable "extra_fqdn" { + type = list(string) + description = "Additional fqdn's must be in the same hosted zone as `domain_name`." + default = [] +} + +########################################## +# Load Balancer # +########################################## +variable "ssl_policy" { + type = string + default = "ELBSecurityPolicy-FS-1-2-Res-2020-10" + description = "SSL policy to use on ALB listener" +} + +variable "acm_certificate_arn" { + type = string + default = null + description = "The ARN of an existing ACM certificate." +} + +variable "allowed_inbound_cidr" { + description = "CIDRs allowed to access wandb-server." + nullable = false + type = list(string) +} + +variable "allowed_inbound_ipv6_cidr" { + description = "CIDRs allowed to access wandb-server." + nullable = false + type = list(string) +} + + +########################################## +# KMS # +########################################## +variable "kms_key_alias" { + type = string + description = "KMS key alias for AWS KMS Customer managed key." + default = null +} + +variable "kms_key_deletion_window" { + type = number + description = "Duration in days to destroy the key after it is deleted. Must be between 7 and 30 days." + default = 7 +} + +variable "kms_key_policy" { + type = string + description = "The policy that will define the permissions for the kms key." + default = "" +} + +########################################## +# Network # +########################################## +variable "create_vpc" { + type = bool + description = "Boolean indicating whether to deploy a VPC (true) or not (false)." + default = false +} + +variable "network_id" { + default = "" + description = "The identity of the VPC in which resources will be deployed." + type = string +} + +variable "network_private_subnets" { + default = [] + description = "A list of the identities of the private subnetworks in which resources will be deployed." + type = list(string) +} + +variable "network_public_subnets" { + default = [] + description = "A list of the identities of the public subnetworks in which resources will be deployed." + type = list(string) +} + +variable "network_database_subnets" { + default = [] + description = "A list of the identities of the database subnetworks in which resources will be deployed." + type = list(string) +} + +variable "network_elasticache_subnets" { + default = [] + description = "A list of the identities of the subnetworks in which elasticache resources will be deployed." + type = list(string) +} + +variable "network_cidr" { + type = string + description = "CIDR block for VPC." + default = "10.10.0.0/16" +} + +variable "network_public_subnet_cidrs" { + type = list(string) + description = "List of private subnet CIDR ranges to create in VPC." + default = ["10.10.0.0/24", "10.10.1.0/24"] +} + +variable "network_private_subnet_cidrs" { + type = list(string) + description = "List of private subnet CIDR ranges to create in VPC." + default = ["10.10.10.0/24", "10.10.11.0/24"] +} + +variable "network_database_subnet_cidrs" { + type = list(string) + description = "List of private subnet CIDR ranges to create in VPC." + default = ["10.10.20.0/24", "10.10.21.0/24"] +} + +variable "network_elasticache_subnet_cidrs" { + type = list(string) + description = "List of private subnet CIDR ranges to create in VPC." + default = ["10.10.30.0/24", "10.10.31.0/24"] +} + +variable "private_link_allowed_account_ids" { + description = "List of AWS account IDs allowed to access the VPC Endpoint Service" + type = list(string) + default = [] +} + +########################################## +# EKS Cluster # +########################################## +variable "eks_cluster_version" { + description = "EKS cluster kubernetes version" + nullable = false + type = string +} +variable "kubernetes_alb_internet_facing" { + type = bool + description = "Indicates whether or not the ALB controlled by the Amazon ALB ingress controller is internet-facing or internal." + default = true +} + +variable "kubernetes_alb_subnets" { + type = list(string) + description = "List of subnet ID's the ALB will use for ingress traffic." + default = [] +} + +variable "kubernetes_public_access" { + type = bool + description = "Indicates whether or not the Amazon EKS public API server endpoint is enabled." + default = true +} + + +variable "kubernetes_public_access_cidrs" { + description = "List of CIDR blocks which can access the Amazon EKS public API server endpoint." + type = list(string) + default = [] +} + +variable "kubernetes_map_accounts" { + description = "Additional AWS account numbers to add to the aws-auth configmap." + type = list(string) + default = [] +} + +variable "kubernetes_map_roles" { + description = "Additional IAM roles to add to the aws-auth configmap." + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) + default = [] +} + +variable "kubernetes_map_users" { + description = "Additional IAM users to add to the aws-auth configmap." + type = list(object({ + userarn = string + username = string + groups = list(string) + })) + default = [] +} + +variable "kubernetes_instance_types" { + description = "EC2 Instance type for primary node group." + type = list(string) + default = ["m5.large"] +} + +variable "kubernetes_node_count" { + description = "Number of nodes" + type = number + default = 2 +} + +variable "eks_policy_arns" { + type = list(string) + description = "Additional IAM policy to apply to the EKS cluster" + default = [] +} + +variable "system_reserved_cpu_millicores" { + description = "(Optional) The amount of 'system-reserved' CPU millicores to pass to the kubelet. For example: 100. A value of -1 disables the flag." + type = number + default = 70 +} + +variable "system_reserved_memory_megabytes" { + description = "(Optional) The amount of 'system-reserved' memory in megabytes to pass to the kubelet. For example: 100. A value of -1 disables the flag." + type = number + default = 100 +} + +variable "system_reserved_ephemeral_megabytes" { + description = "(Optional) The amount of 'system-reserved' ephemeral storage in megabytes to pass to the kubelet. For example: 1000. A value of -1 disables the flag." + type = number + default = 750 +} + +variable "system_reserved_pid" { + description = "(Optional) The amount of 'system-reserved' process ids [pid] to pass to the kubelet. For example: 1000. A value of -1 disables the flag." + type = number + default = 500 +} + +variable "aws_loadbalancer_controller_tags" { + description = "(Optional) A map of AWS tags to apply to all resources managed by the load balancer controller" + type = map(string) + default = {} +} + +########################################## +# External Bucket # +########################################## +# Most users will not need these settings. They are ment for users who want a +# bucket and sqs that are in a different account. +variable "create_bucket" { + type = bool + default = true +} + +variable "bucket_name" { + type = string + default = "" +} + +variable "bucket_kms_key_arn" { + type = string + description = "The Amazon Resource Name of the KMS key with which S3 storage bucket objects will be encrypted." + default = "" +} + +########################################## +# Redis # +########################################## +variable "create_elasticache" { + type = bool + description = "Boolean indicating whether to provision an elasticache instance (true) or not (false)." + default = true +} + +variable "elasticache_node_type" { + description = "The type of the redis cache node to deploy" + type = string + default = "cache.t2.medium" +} + +# ########################################## +# # Weights & Biases # +# ########################################## +variable "license" { + type = string + description = "Weights & Biases license key." +} + +variable "other_wandb_env" { + type = map(any) + description = "Extra environment variables for W&B" + default = {} +} + +variable "weave_wandb_env" { + type = map(string) + description = "Extra environment variables for W&B" + default = {} +} + +variable "app_wandb_env" { + type = map(string) + description = "Extra environment variables for W&B" + default = {} +} + +variable "parquet_wandb_env" { + type = map(string) + description = "Extra environment variables for W&B" + default = {} +}