From ddfdaa1f2f079a4d0e32a482f99cfe4035366296 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Wed, 13 Mar 2024 09:54:15 -0400 Subject: [PATCH 01/33] remove unnecessary variables --- variables.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/variables.tf b/variables.tf index e9ae8c5..0b791dc 100644 --- a/variables.tf +++ b/variables.tf @@ -326,12 +326,6 @@ variable "enable_self_managed_nodes" { description = "Enables self managed nodes" } -variable "enable_eks_managed_nodes" { - type = bool - default = false - description = "Enables eks managed nodes" -} - variable "force_update_version" { type = bool default = true From 4f7b00ed512fbd1a19ea4c2032ea1febc328c841 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Wed, 13 Mar 2024 09:54:25 -0400 Subject: [PATCH 02/33] update for bottlerocket changes --- main.tf | 145 ++++++++++---------------------------------------------- 1 file changed, 25 insertions(+), 120 deletions(-) diff --git a/main.tf b/main.tf index f272cd6..d7f222f 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,8 @@ locals { - name = var.cluster_name - cluster_version = var.cluster_version - hoplimit_metadata = var.enable_hoplimit ? { http_put_response_hop_limit = 1 } : {} + name = var.cluster_name + cluster_version = var.cluster_version + hoplimit_metadata = var.enable_hoplimit ? { http_put_response_hop_limit = 1 } : {} + is_bottlerocket_ami = contains(split("-", data.aws_ami.eks_ami.name), "bottlerocket") } data "aws_ami" "eks_ami" { @@ -21,114 +22,13 @@ data "aws_security_groups" "delete_ebs_volumes_lambda_security_group" { # EKS Module ################################################################################ locals { - custom_node_pools = { for k, v in merge({ general = var.general_node_pool }, var.custom_node_pools) : k => { - name = "${var.cluster_name}-${k}" - subnet_ids = coalescelist(try(v.subnet_ids, []), var.host_subnets, var.private_subnets) - ami_id = data.aws_ami.eks_ami.id - iam_role_path = var.iam_role_path - iam_role_permissions_boundary = var.iam_role_permissions_boundary - - instance_type = v.instance_type - desired_size = v.desired_size - max_size = v.max_size - min_size = v.min_size - - ## Define custom lines to the user_data script. Separate commands with \n - pre_bootstrap_user_data = try(v.pre_bootstrap_user_data, "sysctl -w net.ipv4.ip_forward=1\n") - post_bootstrap_user_data = try(v.post_bootstrap_user_data, "") - bootstrap_extra_args = join(" ", - ["--kubelet-extra-args '--node-labels=${k}=true", try(v.extra_args, "")], - ["--pod-max-pids=1000"], - [for label_key, label_value in try(v.labels, {}) : "--node-labels=${label_key}=${label_value}"], - [for taint_key, taint_value in try(v.taints, {}) : "--register-with-taints=${taint_key}=${taint_value}"], - ["'"] - ) - create_security_group = false - block_device_mappings = [ - { - device_name = "/dev/xvda" - ebs = { - volume_size = try(v.volume_size, "300") - volume_type = try(v.volume_type, "gp3") - delete_on_termination = try(v.volume_delete_on_termination, true) - encrypted = true - } - } - ] - - # On the general node group or any node group labeled "general", attach target groups - target_group_arns = (k == "general" || contains(keys(try(v.labels, {})), "general")) ? concat( - [aws_lb_target_group.batcave_alb_https.arn], - var.create_alb_proxy ? [aws_lb_target_group.batcave_alb_proxy_https[0].arn] : [], - var.create_alb_shared ? [aws_lb_target_group.batcave_alb_shared_https[0].arn] : [] - ) : null - - tags = merge(var.tags, local.instance_tags, try(v.tags, null)) - - metadata_options = merge(local.hoplimit_metadata, try(v.metadata_options, {})) + # Schedule config + create_schedule_startup = var.node_schedule_startup_hour >= 0 || var.node_schedule_startup_cron != "" + create_schedule_shutdown = var.node_schedule_shutdown_hour >= 0 || var.node_schedule_shutdown_cron != "" + create_schedule = local.create_schedule_startup || local.create_schedule_shutdown + node_schedule_shutdown_cron = var.node_schedule_shutdown_cron != "" ? var.node_schedule_shutdown_cron : "0 ${var.node_schedule_shutdown_hour} * * *" + node_schedule_startup_cron = var.node_schedule_startup_cron != "" ? var.node_schedule_startup_cron : "0 ${var.node_schedule_startup_hour} * * 1-5" - ## Tags that are applied _ONLY_ to the ASG resource and not propagated to the nodes - ## All the "tags" var will be applied to both ASG and Propagated out to the nodes - autoscaling_group_tags = merge( - { - "k8s.io/cluster-autoscaler/enabled" = "true", - "k8s.io/cluster-autoscaler/${var.cluster_name}" = var.cluster_name - }, - # Taint tags for Cluster Autoscaler hints - try({ for taint_key, taint_value in v.taints : "k8s.io/cluster-autoscaler/node-template/taint/${taint_key}" => taint_value }, {}), - # Label tags for Cluster Autoscaler hints - { "k8s.io/cluster-autoscaler/node-template/label/${k}" = "true" }, - try({ for label_key, label_value in v.labels : "k8s.io/cluster-autoscaler/node-template/label/${label_key}" => label_value }, {}), - var.autoscaling_group_tags, - ) - enabled_metrics = [ - "GroupAndWarmPoolDesiredCapacity", - "GroupAndWarmPoolTotalCapacity", - "GroupDesiredCapacity", - "GroupInServiceCapacity", - "GroupInServiceInstances", - "GroupMaxSize", - "GroupMinSize", - "GroupPendingCapacity", - "GroupPendingInstances", - "GroupStandbyCapacity", - "GroupStandbyInstances", - "GroupTerminatingCapacity", - "GroupTerminatingInstances", - "GroupTotalCapacity", - "GroupTotalInstances", - "WarmPoolDesiredCapacity", - "WarmPoolMinSize", - "WarmPoolPendingCapacity", - "WarmPoolTerminatingCapacity", - "WarmPoolTotalCapacity", - "WarmPoolWarmedCapacity", - ] - create_schedule = var.node_schedule_shutdown_hour >= 0 || var.node_schedule_startup_hour >= 0 - schedules = merge( - var.node_schedule_shutdown_hour < 0 ? {} : { - shutdown = { - min_size = 0 - max_size = 0 - desired_size = 0 - time_zone = var.node_schedule_timezone - recurrence = "0 ${var.node_schedule_shutdown_hour} * * *" - } - }, - var.node_schedule_startup_hour < 0 ? {} : { - startup = { - min_size = v.min_size - max_size = v.max_size - desired_size = v.desired_size - time_zone = var.node_schedule_timezone - recurrence = "0 ${var.node_schedule_startup_hour} * * 1-5" - } - } - ) - - ## https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group#instance_refresh - instance_refresh = lookup(v, "instance_refresh", {}) - } } instance_policy_tags = var.enable_ssm_patching ? { "Patch Group" = var.ssm_tag_patch_group, "Patch Window" = var.ssm_tag_patch_window } : {} instance_tags = merge(local.instance_policy_tags, var.instance_tags) @@ -162,15 +62,15 @@ locals { iam_role_permissions_boundary = var.iam_role_permissions_boundary ami_id = data.aws_ami.eks_ami.id - + # Added for Bottlerocket use_custom_launch_template = try(v.use_custom_launch_template, true) - ami_type = var.platform == "bottlerocket" ? "BOTTLEROCKET_x86_64" : "AL2_x86_64" - platform = try(var.platform, "linux") - bootstrap_extra_args = <<-EOT + ami_type = var.platform == "bottlerocket" ? "BOTTLEROCKET_x86_64" : "AL2_x86_64" + platform = try(var.platform, "linux") + bootstrap_extra_args = <<-EOT # settings.kubernetes section from bootstrap_extra_args in default template pod-pids-limit = 1000 - + # The admin host container provides SSH access and runs with "superpowers". # It is disabled by default, but can be disabled explicitly. [settings.host-containers.admin] @@ -202,7 +102,7 @@ locals { max_size = v.max_size desired_size = v.desired_size - block_device_mappings = [ + base_block_device_mappings = [ { device_name = "/dev/xvda" ebs = { @@ -223,6 +123,14 @@ locals { } ] + block_device_mappings = local.is_bottlerocket_ami ? [ + for index, block_device in v.base_block_device_mappings : + index > 0 ? { + device_name = idx == 1 ? "/dev/xvda" : block_device.device_name + ebs = block_device.ebs + } : null + ] : v.base_block_device_mappings + ## Define custom lines to the user_data script. Separate commands with \n @@ -316,9 +224,6 @@ module "eks" { ## CLUSTER Addons cluster_addons = {} - # Worker groups (using Launch Configurations) - self_managed_node_groups = var.enable_self_managed_nodes ? local.custom_node_pools : {} - # apply any global tags to the cluster itself cluster_tags = var.tags } @@ -327,7 +232,7 @@ module "eks_managed_node_groups" { source = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group" version = "19.21.0" - for_each = var.enable_eks_managed_nodes ? local.eks_node_pools : {} + for_each = local.eks_node_pools name = each.value.name cluster_name = each.value.cluster_name From f0318f542a9fccc4ad5bdd71ca6a547d32371896 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Fri, 15 Mar 2024 10:28:44 -0400 Subject: [PATCH 03/33] fix missing vars --- variables.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/variables.tf b/variables.tf index 0b791dc..f7c1da0 100644 --- a/variables.tf +++ b/variables.tf @@ -272,16 +272,31 @@ variable "ami_regex_override" { default = "" type = string } + +variable "node_schedule_shutdown_cron" { + type = string + default = "" + description = "The cron schedule for the cluster to be shutdown. If left empty, the cluster will not be stopped. Will run every day otherwise." +} + +variable "node_schedule_startup_cron" { + type = string + default = "" + description = "The cron schedule for the cluster to be restarted. If left empty, the cluster will not be restarted after shutdown. Will run every weekday otherwise." +} + variable "node_schedule_shutdown_hour" { type = number default = -1 description = "The hour of the day (0-23) the cluster should be shutdown. If left empty, the cluster will not be stopped. Will run every day otherwise." } + variable "node_schedule_startup_hour" { type = number default = -1 description = "The hour of the day (0-23) the cluster should be restarted. If left empty, the cluster will not be restarted after shutdown. Will run every weekday otherwise." } + variable "node_schedule_timezone" { type = string default = "America/New_York" From e9c109ee2524d45a66ead590f07df901f202ffa9 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Fri, 15 Mar 2024 10:54:11 -0400 Subject: [PATCH 04/33] fix for loop ref --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index d7f222f..114334e 100644 --- a/main.tf +++ b/main.tf @@ -126,7 +126,7 @@ locals { block_device_mappings = local.is_bottlerocket_ami ? [ for index, block_device in v.base_block_device_mappings : index > 0 ? { - device_name = idx == 1 ? "/dev/xvda" : block_device.device_name + device_name = index == 1 ? "/dev/xvda" : block_device.device_name ebs = block_device.ebs } : null ] : v.base_block_device_mappings From 863b783c962882ad6a9ecbe82845f66aaba88203 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Fri, 15 Mar 2024 11:03:34 -0400 Subject: [PATCH 05/33] remove unneeded var reference --- main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 114334e..fa5e613 100644 --- a/main.tf +++ b/main.tf @@ -464,7 +464,7 @@ resource "aws_iam_role" "cosign" { #} resource "aws_autoscaling_attachment" "eks_managed_node_groups_alb_attachment" { - for_each = var.enable_eks_managed_nodes ? { for np in local.eks_node_pools : np.name => np } : {} + for_each = { for np in local.eks_node_pools : np.name => np } autoscaling_group_name = try(module.eks_managed_node_groups[each.value.group_name].node_group_autoscaling_group_names[0], "") lb_target_group_arn = aws_lb_target_group.batcave_alb_https.arn @@ -475,7 +475,7 @@ resource "aws_autoscaling_attachment" "eks_managed_node_groups_alb_attachment" { } resource "aws_autoscaling_attachment" "eks_managed_node_groups_proxy_attachment" { - for_each = var.create_alb_proxy && var.enable_eks_managed_nodes ? { for np in local.eks_node_pools : np.name => np } : {} + for_each = var.create_alb_proxy ? { for np in local.eks_node_pools : np.name => np } : {} autoscaling_group_name = try(module.eks_managed_node_groups[each.value.group_name].node_group_autoscaling_group_names[0], "") lb_target_group_arn = var.create_alb_proxy ? aws_lb_target_group.batcave_alb_proxy_https[0].arn : null @@ -486,7 +486,7 @@ resource "aws_autoscaling_attachment" "eks_managed_node_groups_proxy_attachment" } resource "aws_autoscaling_attachment" "eks_managed_node_groups_shared_attachment" { - for_each = var.create_alb_shared && var.enable_eks_managed_nodes ? { for np in local.eks_node_pools : np.name => np } : {} + for_each = var.create_alb_shared ? { for np in local.eks_node_pools : np.name => np } : {} autoscaling_group_name = try(module.eks_managed_node_groups[each.value.group_name].node_group_autoscaling_group_names[0], "") lb_target_group_arn = var.create_alb_shared ? aws_lb_target_group.batcave_alb_shared_https[0].arn : null From d8aabb75a90b313683a81a78e7ddf752b88948ef Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Fri, 15 Mar 2024 11:19:56 -0400 Subject: [PATCH 06/33] abstract out volume params --- main.tf | 50 +++++++++++++++++++++++++++----------------------- variables.tf | 8 ++++---- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/main.tf b/main.tf index fa5e613..465ee13 100644 --- a/main.tf +++ b/main.tf @@ -52,6 +52,31 @@ locals { # EKS Fully managed nodes ################################################################################ locals { + volume_size = try(var.node_volume_size, 300) + volume_type = try(var.node_volume_type, "gp3") + volume_delete_on_termination = try(var.node_volume_delete_on_termination, true) + + base_block_device_mappings = [ + { + device_name = "/dev/xvda" + ebs = { + volume_size = "5" + volume_type = "gp3" + delete_on_termination = true + encrypted = true + } + }, + { + device_name = "/dev/xvdb" + ebs = { + volume_size = try(local.volume_size, "300") + volume_type = try(local.volume_type, "gp3") + delete_on_termination = try(local.volume_delete_on_termination, true) + encrypted = true + } + } + ] + eks_node_pools = { for k, v in merge({ general = var.general_node_pool }, var.custom_node_pools) : k => { group_name = k name = "${var.cluster_name}-${k}" @@ -102,34 +127,13 @@ locals { max_size = v.max_size desired_size = v.desired_size - base_block_device_mappings = [ - { - device_name = "/dev/xvda" - ebs = { - volume_size = "5" - volume_type = "gp3" - delete_on_termination = true - encrypted = true - } - }, - { - device_name = "/dev/xvdb" - ebs = { - volume_size = try(v.volume_size, "300") - volume_type = try(v.volume_type, "gp3") - delete_on_termination = try(v.volume_delete_on_termination, true) - encrypted = true - } - } - ] - block_device_mappings = local.is_bottlerocket_ami ? [ - for index, block_device in v.base_block_device_mappings : + for index, block_device in local.base_block_device_mappings : index > 0 ? { device_name = index == 1 ? "/dev/xvda" : block_device.device_name ebs = block_device.ebs } : null - ] : v.base_block_device_mappings + ] : local.base_block_device_mappings diff --git a/variables.tf b/variables.tf index f7c1da0..3ea9fdb 100644 --- a/variables.tf +++ b/variables.tf @@ -26,10 +26,10 @@ variable "general_node_pool" { default = { instance_type = "c5.2xlarge" # ami_type = "BOTTLEROCKET_x86_64" - desired_size = 3 - max_size = 5 - min_size = 2 - use_custom_launch_template = false + desired_size = 3 + max_size = 5 + min_size = 2 + use_custom_launch_template = false # Map of label flags for kubelets. labels = { general = "true" } taints = {} From 073a25d39d86df5abaf2784e0ce53607497f0667 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Fri, 15 Mar 2024 11:26:13 -0400 Subject: [PATCH 07/33] fix vol vars --- main.tf | 10 +++------- variables.tf | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/main.tf b/main.tf index 465ee13..10e0b3b 100644 --- a/main.tf +++ b/main.tf @@ -52,10 +52,6 @@ locals { # EKS Fully managed nodes ################################################################################ locals { - volume_size = try(var.node_volume_size, 300) - volume_type = try(var.node_volume_type, "gp3") - volume_delete_on_termination = try(var.node_volume_delete_on_termination, true) - base_block_device_mappings = [ { device_name = "/dev/xvda" @@ -69,9 +65,9 @@ locals { { device_name = "/dev/xvdb" ebs = { - volume_size = try(local.volume_size, "300") - volume_type = try(local.volume_type, "gp3") - delete_on_termination = try(local.volume_delete_on_termination, true) + volume_size = var.node_volume_size + volume_type = var.node_volume_type + delete_on_termination = var.node_volume_delete_on_termination encrypted = true } } diff --git a/variables.tf b/variables.tf index 3ea9fdb..4aee46e 100644 --- a/variables.tf +++ b/variables.tf @@ -122,6 +122,24 @@ variable "node_https_ingress_cidr_blocks" { type = list(string) } +variable "node_volume_size" { + description = "The size of the volume to use for the nodes. Defaults to 20" + default = 300 + type = number +} + +variable "node_volume_type" { + description = "The type of volume to use for the nodes. Defaults to gp2" + default = "gp3" + type = string +} + +variable "node_volume_delete_on_termination" { + description = "Whether the volume should be deleted when the node is terminated. Defaults to true" + default = true + type = bool +} + variable "alb_restricted_hosts" { type = set(string) description = "A list of allowable host for private alb" From 15e1edacf0513920917d42639aeb69f0cb532ebf Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Mon, 18 Mar 2024 10:23:27 -0400 Subject: [PATCH 08/33] change block device mappings logic --- main.tf | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/main.tf b/main.tf index 10e0b3b..3130215 100644 --- a/main.tf +++ b/main.tf @@ -56,7 +56,7 @@ locals { { device_name = "/dev/xvda" ebs = { - volume_size = "5" + volume_size = "8" volume_type = "gp3" delete_on_termination = true encrypted = true @@ -123,13 +123,18 @@ locals { max_size = v.max_size desired_size = v.desired_size + # This is dynamically creating the block device mappings based on the AMI type block_device_mappings = local.is_bottlerocket_ami ? [ - for index, block_device in local.base_block_device_mappings : - index > 0 ? { - device_name = index == 1 ? "/dev/xvda" : block_device.device_name - ebs = block_device.ebs - } : null - ] : local.base_block_device_mappings + { + device_name = "/dev/xvda" + ebs = { + volume_size = var.node_volume_size + volume_type = var.node_volume_type + delete_on_termination = var.node_volume_delete_on_termination + encrypted = true + } + } + ] : local.base_block_device_mappings From a4f0f1e601947ad9e5892b988dfba8e3af3fad51 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Mon, 18 Mar 2024 10:50:12 -0400 Subject: [PATCH 09/33] update logic again --- main.tf | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/main.tf b/main.tf index 3130215..aa919d5 100644 --- a/main.tf +++ b/main.tf @@ -124,17 +124,11 @@ locals { desired_size = v.desired_size # This is dynamically creating the block device mappings based on the AMI type - block_device_mappings = local.is_bottlerocket_ami ? [ + block_device_mappings = local.is_bottlerocket_ami ? local.base_block_device_mappings : { device_name = "/dev/xvda" - ebs = { - volume_size = var.node_volume_size - volume_type = var.node_volume_type - delete_on_termination = var.node_volume_delete_on_termination - encrypted = true - } + ebs = local.base_block_device_mappings[1].ebs } - ] : local.base_block_device_mappings From a0a19e56bce520d514c2e181378f26523d96a20a Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Mon, 18 Mar 2024 11:09:30 -0400 Subject: [PATCH 10/33] fix if statement --- main.tf | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/main.tf b/main.tf index aa919d5..5f6935f 100644 --- a/main.tf +++ b/main.tf @@ -124,11 +124,12 @@ locals { desired_size = v.desired_size # This is dynamically creating the block device mappings based on the AMI type - block_device_mappings = local.is_bottlerocket_ami ? local.base_block_device_mappings : - { - device_name = "/dev/xvda" - ebs = local.base_block_device_mappings[1].ebs - } + block_device_mappings = local.is_bottlerocket_ami ? local.base_block_device_mappings : [ + { + device_name = "/dev/xvda" + ebs = local.base_block_device_mappings[1].ebs + } + ] From 8d2fef3a9e71b97bb8880aa0b6bd402f048868c6 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Tue, 19 Mar 2024 14:17:34 -0400 Subject: [PATCH 11/33] update ami to default to cms image --- main.tf | 4 ++-- variables.tf | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 5f6935f..b8c4525 100644 --- a/main.tf +++ b/main.tf @@ -7,8 +7,8 @@ locals { data "aws_ami" "eks_ami" { most_recent = true - name_regex = var.ami_regex_override == "" ? "^bottlerocket-aws-k8s-1.27-x86_64-v1.17.0" : var.ami_regex_override - owners = ["092701018921"] + name_regex = var.use_bottlerocket ? "^bottlerocket-aws-k8s-${var.cluster_version}-x86_64-v1.17.0" : (var.ami_regex_override == "" ? "^amzn2-eks-${var.cluster_version}-gi-${var.ami_date}" : var.ami_regex_override) + owners = var.use_bottlerocket ? ["092701018921"] : (var.ami_owner_override == [""] ? ["743302140042"] : var.ami_owner_override) } data "aws_security_groups" "delete_ebs_volumes_lambda_security_group" { diff --git a/variables.tf b/variables.tf index 4aee46e..5b2863a 100644 --- a/variables.tf +++ b/variables.tf @@ -394,3 +394,15 @@ variable "ssm_tag_patch_window" { default = "ITOPS-Wave1-Non-Mktplc-DevTestImpl-MW" description = "SSM Patching window for instances. For more information: https://cloud.cms.gov/patching-prerequisites" } + +variable "ami_owner_override" { + type = list(string) + default = [""] + description = "AMI owner override" +} + +variable "use_bottlerocket" { + type = bool + default = false + description = "Use Bottlerocket" +} From f0b25c3d6a983aa45941533a57301ed0930574cb Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Thu, 21 Mar 2024 10:14:13 -0400 Subject: [PATCH 12/33] test ami owner change --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index b8c4525..9e9bacb 100644 --- a/main.tf +++ b/main.tf @@ -8,7 +8,7 @@ locals { data "aws_ami" "eks_ami" { most_recent = true name_regex = var.use_bottlerocket ? "^bottlerocket-aws-k8s-${var.cluster_version}-x86_64-v1.17.0" : (var.ami_regex_override == "" ? "^amzn2-eks-${var.cluster_version}-gi-${var.ami_date}" : var.ami_regex_override) - owners = var.use_bottlerocket ? ["092701018921"] : (var.ami_owner_override == [""] ? ["743302140042"] : var.ami_owner_override) + owners = var.use_bottlerocket ? ["092701018921"] : ["743302140042"] } data "aws_security_groups" "delete_ebs_volumes_lambda_security_group" { From 42ec476c5c5b912c2566dc091ccec6bdef61ec03 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Thu, 21 Mar 2024 10:18:36 -0400 Subject: [PATCH 13/33] update ami owner var logic --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 9e9bacb..194c9e4 100644 --- a/main.tf +++ b/main.tf @@ -8,7 +8,7 @@ locals { data "aws_ami" "eks_ami" { most_recent = true name_regex = var.use_bottlerocket ? "^bottlerocket-aws-k8s-${var.cluster_version}-x86_64-v1.17.0" : (var.ami_regex_override == "" ? "^amzn2-eks-${var.cluster_version}-gi-${var.ami_date}" : var.ami_regex_override) - owners = var.use_bottlerocket ? ["092701018921"] : ["743302140042"] + owners = var.use_bottlerocket ? ["092701018921"] : (length(var.ami_owner_override) > 0 && var.ami_owner_override[0] != "" ? var.ami_owner_override : ["743302140042"]) } data "aws_security_groups" "delete_ebs_volumes_lambda_security_group" { From bb4f0eb9bcfd8cb110c4429d2337356e2e1e2779 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Thu, 21 Mar 2024 10:46:35 -0400 Subject: [PATCH 14/33] update for bottlerocket config --- main.tf | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 194c9e4..67b8a94 100644 --- a/main.tf +++ b/main.tf @@ -75,9 +75,9 @@ locals { eks_node_pools = { for k, v in merge({ general = var.general_node_pool }, var.custom_node_pools) : k => { group_name = k - name = "${var.cluster_name}-${k}" - cluster_name = local.name - cluster_version = local.cluster_version + name = "${module.eks.cluster_name}-${k}" + cluster_name = module.eks.cluster_name + cluster_version = module.eks.cluster_version iam_role_path = var.iam_role_path iam_role_permissions_boundary = var.iam_role_permissions_boundary @@ -89,6 +89,16 @@ locals { ami_type = var.platform == "bottlerocket" ? "BOTTLEROCKET_x86_64" : "AL2_x86_64" platform = try(var.platform, "linux") bootstrap_extra_args = <<-EOT + # Base settings for bottlerocket + [settings.kubernetes] + cluster-name = "${module.eks.cluster_name}" + api-server = "${module.eks.cluster_endpoint}" + cluster-certificate = "${module.eks.cluster_certificate_authority_data}" + + # Set autoscaling wait + [settings.autoscaling] + should-wait = true + # settings.kubernetes section from bootstrap_extra_args in default template pod-pids-limit = 1000 From bccded1d1ea456713583b7d1e8baeb7131bd233f Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Fri, 22 Mar 2024 08:57:46 -0400 Subject: [PATCH 15/33] add bottlerocket toml file --- main.tf | 56 +++++++++------------------------ templates/bottlerocket.toml.tpl | 35 +++++++++++++++++++++ variables.tf | 12 +++++++ 3 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 templates/bottlerocket.toml.tpl diff --git a/main.tf b/main.tf index 9e04fe7..03e2dd6 100644 --- a/main.tf +++ b/main.tf @@ -73,6 +73,19 @@ locals { } ] + node_labels = merge( + var.node_labels + ) + + bottlerocket_bootstrap_template = templatefile("${path.module}/templates/bottlerocket.toml.tpl", { + cluster_name = var.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_ca_data = module.eks.cluster_certificate_authority_data + max_namespaces = 10000 + node_labels = join("\n", [for label, value in local.node_labels : "\"${label}\" = \"${value}\""]) + node_taints = join("\n", [for taint, value in var.node_taints : "\"${taint}\" = \"${value}\""]) + }) + eks_node_pools = { for k, v in merge({ general = var.general_node_pool }, var.custom_node_pools) : k => { group_name = k name = "${module.eks.cluster_name}-${k}" @@ -88,46 +101,7 @@ locals { use_custom_launch_template = try(v.use_custom_launch_template, true) ami_type = var.platform == "bottlerocket" ? "BOTTLEROCKET_x86_64" : "AL2_x86_64" platform = try(var.platform, "linux") - bootstrap_extra_args = <<-EOT - # Base settings for bottlerocket - [settings.kubernetes] - cluster-name = "${module.eks.cluster_name}" - api-server = "${module.eks.cluster_endpoint}" - cluster-certificate = "${module.eks.cluster_certificate_authority_data}" - - # Set autoscaling wait - [settings.autoscaling] - should-wait = true - - # settings.kubernetes section from bootstrap_extra_args in default template - pod-pids-limit = 1000 - - # The admin host container provides SSH access and runs with "superpowers". - # It is disabled by default, but can be disabled explicitly. - [settings.host-containers.admin] - enabled = false - - # The control host container provides out-of-band access via SSM. - # It is enabled by default, and can be disabled if you do not expect to use SSM. - # This could leave you with no way to access the API and change settings on an existing node! - [settings.host-containers.control] - enabled = true - - # extra args added - [settings.kernel] - lockdown = "integrity" - - [settings.kernel.sysctl] - "user.max_user_namespaces" = "10000" - - [settings.kubernetes.node-labels] - # label1 = "foo" - # label2 = "bar" - - [settings.kubernetes.node-taints] - # dedicated = "experimental:PreferNoSchedule" - # special = "true:NoSchedule" - EOT + bootstrap_extra_args = local.bottlerocket_bootstrap_template subnet_ids = coalescelist(try(v.subnet_ids, []), var.host_subnets, var.private_subnets) @@ -139,7 +113,7 @@ locals { block_device_mappings = local.is_bottlerocket_ami ? local.base_block_device_mappings : [ { device_name = "/dev/xvda" - ebs = local.base_block_device_mappings[1].ebs + ebs = local.base_block_device_mappings[1].ebs } ] diff --git a/templates/bottlerocket.toml.tpl b/templates/bottlerocket.toml.tpl new file mode 100644 index 0000000..9bee4a0 --- /dev/null +++ b/templates/bottlerocket.toml.tpl @@ -0,0 +1,35 @@ +[settings.kubernetes] +cluster-name = "${cluster_name}" +api-server = "${cluster_endpoint}" +cluster-certificate = "${cluster_ca_data}" + +# Set autoscaling wait +[settings.autoscaling] +should-wait = true + +# settings.kubernetes section from bootstrap_extra_args in default template +pod-pids-limit = 1000 + +# The admin host container provides SSH access and runs with "superpowers". +# It is disabled by default, but can be disabled explicitly. +[settings.host-containers.admin] +enabled = false + +# The control host container provides out-of-band access via SSM. +# It is enabled by default, and can be disabled if you do not expect to use SSM. +# This could leave you with no way to access the API and change settings on an existing node! +[settings.host-containers.control] +enabled = true + +# extra args added +[settings.kernel] +lockdown = "integrity" + +[settings.kernel.sysctl] +"user.max_user_namespaces" = "${max_namespaces}" + +[settings.kubernetes.node-labels] +${node_labels} + +[settings.kubernetes.node-taints] +${node_taints} diff --git a/variables.tf b/variables.tf index 5b2863a..c710d2c 100644 --- a/variables.tf +++ b/variables.tf @@ -406,3 +406,15 @@ variable "use_bottlerocket" { default = false description = "Use Bottlerocket" } + +variable "node_labels" { + description = "The labels to apply to the EKS nodes" + type = map(string) + default = {} +} + +variable "node_taints" { + description = "The taints to apply to the EKS nodes" + type = map(string) + default = {} +} From afd29f7f10f99db9f555722702f86190f5c74719 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Fri, 22 Mar 2024 09:40:24 -0400 Subject: [PATCH 16/33] cluster info already injected --- templates/bottlerocket.toml.tpl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/templates/bottlerocket.toml.tpl b/templates/bottlerocket.toml.tpl index 9bee4a0..9e9eeac 100644 --- a/templates/bottlerocket.toml.tpl +++ b/templates/bottlerocket.toml.tpl @@ -1,8 +1,3 @@ -[settings.kubernetes] -cluster-name = "${cluster_name}" -api-server = "${cluster_endpoint}" -cluster-certificate = "${cluster_ca_data}" - # Set autoscaling wait [settings.autoscaling] should-wait = true From e3a53a877f1180ad232abee357458496325f5afd Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Fri, 22 Mar 2024 09:55:54 -0400 Subject: [PATCH 17/33] more updates for br --- main.tf | 8 ++------ variables.tf | 6 ------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/main.tf b/main.tf index 03e2dd6..6e336c6 100644 --- a/main.tf +++ b/main.tf @@ -99,8 +99,8 @@ locals { # Added for Bottlerocket use_custom_launch_template = try(v.use_custom_launch_template, true) - ami_type = var.platform == "bottlerocket" ? "BOTTLEROCKET_x86_64" : "AL2_x86_64" - platform = try(var.platform, "linux") + ami_type = var.use_bottlerocket ? "BOTTLEROCKET_x86_64" : "AL2_x86_64" + platform = var.use_bottlerocket ? "bottlerocket" : "linux" bootstrap_extra_args = local.bottlerocket_bootstrap_template subnet_ids = coalescelist(try(v.subnet_ids, []), var.host_subnets, var.private_subnets) @@ -203,10 +203,6 @@ module "eks" { resources = ["secrets"] } - self_managed_node_group_defaults = { - subnet_ids = coalescelist(var.host_subnets, var.private_subnets) - } - ## CLUSTER Addons cluster_addons = {} diff --git a/variables.tf b/variables.tf index c710d2c..c63eba3 100644 --- a/variables.tf +++ b/variables.tf @@ -14,12 +14,6 @@ variable "ami_date" { type = string } -## Default node group -variable "platform" { - default = "bottlerocket" - type = string -} - variable "general_node_pool" { type = any description = "General node pool, required for hosting core services" From 40264415bd072fabec38f065a7c99fc0ec9f6a7f Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Fri, 22 Mar 2024 11:48:52 -0400 Subject: [PATCH 18/33] move setting in toml --- main.tf | 7 +++---- templates/bottlerocket.toml.tpl | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/main.tf b/main.tf index 6e336c6..63f2653 100644 --- a/main.tf +++ b/main.tf @@ -2,7 +2,6 @@ locals { name = var.cluster_name cluster_version = var.cluster_version hoplimit_metadata = var.enable_hoplimit ? { http_put_response_hop_limit = 1 } : {} - is_bottlerocket_ami = contains(split("-", data.aws_ami.eks_ami.name), "bottlerocket") } data "aws_ami" "eks_ami" { @@ -110,7 +109,7 @@ locals { desired_size = v.desired_size # This is dynamically creating the block device mappings based on the AMI type - block_device_mappings = local.is_bottlerocket_ami ? local.base_block_device_mappings : [ + block_device_mappings = var.use_bottlerocket ? local.base_block_device_mappings : [ { device_name = "/dev/xvda" ebs = local.base_block_device_mappings[1].ebs @@ -217,8 +216,8 @@ module "eks_managed_node_groups" { for_each = local.eks_node_pools name = each.value.name - cluster_name = each.value.cluster_name - cluster_version = each.value.cluster_version + cluster_name = module.eks.cluster_name + cluster_version = module.eks.cluster_version cluster_endpoint = module.eks.cluster_endpoint cluster_auth_base64 = module.eks.cluster_certificate_authority_data create_iam_role = false diff --git a/templates/bottlerocket.toml.tpl b/templates/bottlerocket.toml.tpl index 9e9eeac..94b9c62 100644 --- a/templates/bottlerocket.toml.tpl +++ b/templates/bottlerocket.toml.tpl @@ -1,10 +1,10 @@ +# settings.kubernetes section from bootstrap_extra_args in default template +pod-pids-limit = 1000 + # Set autoscaling wait [settings.autoscaling] should-wait = true -# settings.kubernetes section from bootstrap_extra_args in default template -pod-pids-limit = 1000 - # The admin host container provides SSH access and runs with "superpowers". # It is disabled by default, but can be disabled explicitly. [settings.host-containers.admin] From 0ef94ed2215c011d9f73f74020b5f67d19e48c50 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Fri, 22 Mar 2024 16:35:18 +0000 Subject: [PATCH 19/33] [pre-commit.ci lite] apply automatic fixes --- README.md | 14 +++++++++----- main.tf | 12 +++--------- variables.tf | 31 ++++--------------------------- 3 files changed, 16 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index b936378..456e864 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Launch template with managed groups example +# Launch template with managed groups example This is EKS example using workers custom launch template with managed groups feature in two different ways: @@ -157,8 +157,8 @@ Note that this example may create resources which cost money. Run `terraform des | [alb\_ssl\_security\_policy](#input\_alb\_ssl\_security\_policy) | ALB SSL Security Policy | `string` | `"ELBSecurityPolicy-TLS13-1-2-Res-2021-06"` | no | | [alb\_subnets\_by\_zone](#input\_alb\_subnets\_by\_zone) | n/a | `map(string)` | n/a | yes | | [ami\_date](#input\_ami\_date) | n/a | `string` | `""` | no | +| [ami\_owner\_override](#input\_ami\_owner\_override) | AMI owner override | `list(string)` | 
[
  ""
]
| no | | [ami\_regex\_override](#input\_ami\_regex\_override) | Overrides default AMI lookup regex, which grabs latest AMI matching cluster\_version by default | `string` | `""` | no | -| [autoscaling\_group\_tags](#input\_autoscaling\_group\_tags) | Tags to apply to all autoscaling groups created | `map(any)` | `{}` | no | | [cluster\_additional\_sg\_prefix\_lists](#input\_cluster\_additional\_sg\_prefix\_lists) | n/a | `list(string)` | n/a | yes | | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | 
[
  "api",
  "audit",
  "authenticator",
  "controllerManager",
  "scheduler"
]
| no | | [cluster\_name](#input\_cluster\_name) | n/a | `string` | n/a | yes | @@ -170,14 +170,12 @@ Note that this example may create resources which cost money. Run `terraform des | [create\_cosign\_iam\_role](#input\_create\_cosign\_iam\_role) | Flag to create Cosign IAM role | `bool` | `false` | no | | [custom\_node\_policy\_arns](#input\_custom\_node\_policy\_arns) | Custom node policy arns | `set(string)` | `[]` | no | | [custom\_node\_pools](#input\_custom\_node\_pools) | n/a | `any` | `{}` | no | -| [enable\_eks\_managed\_nodes](#input\_enable\_eks\_managed\_nodes) | Enables eks managed nodes | `bool` | `false` | no | | [enable\_hoplimit](#input\_enable\_hoplimit) | Enables a IMDSv2 hop limit of 1 on all nodes. Defaults to false | `bool` | `false` | no | -| [enable\_self\_managed\_nodes](#input\_enable\_self\_managed\_nodes) | Enables self managed nodes | `bool` | `true` | no | | [enable\_ssm\_patching](#input\_enable\_ssm\_patching) | Enables Systems Manager to patch nodes | `bool` | `false` | no | | [environment](#input\_environment) | n/a | `string` | `"dev"` | no | | [federated\_access\_role](#input\_federated\_access\_role) | Federated access role | `string` | `"ct-ado-batcave-application-admin"` | no | | [force\_update\_version](#input\_force\_update\_version) | Force update version | `bool` | `true` | no | -| [general\_node\_pool](#input\_general\_node\_pool) | General node pool, required for hosting core services | `any` | 
{
  "desired_size": 3,
  "instance_type": "c5.2xlarge",
  "labels": {
    "general": "true"
  },
  "max_size": 5,
  "min_size": 2,
  "taints": {}
}
| no | +| [general\_node\_pool](#input\_general\_node\_pool) | General node pool, required for hosting core services | `any` | 
{
  "desired_size": 3,
  "instance_type": "c5.2xlarge",
  "labels": {
    "general": "true"
  },
  "max_size": 5,
  "min_size": 2,
  "taints": {},
  "use_custom_launch_template": false
}
| no | | [github\_actions\_role](#input\_github\_actions\_role) | Github actions role | `string` | `"batcave-github-actions-role"` | no | | [grant\_delete\_ebs\_volumes\_lambda\_access](#input\_grant\_delete\_ebs\_volumes\_lambda\_access) | When set to true, a cluster role and permissions will be created to grant the delete-ebs-volumes Lambda access to the PersistentVolumes API. | `bool` | `false` | no | | [host\_subnets](#input\_host\_subnets) | Override the ec2 instance subnets. By default, they are launche in private\_subnets, just like the EKS control plane. | `list(any)` | `[]` | no | @@ -186,9 +184,14 @@ Note that this example may create resources which cost money. Run `terraform des | [instance\_tags](#input\_instance\_tags) | Instance custom tags | `map(any)` | `null` | no | | [logging\_bucket](#input\_logging\_bucket) | Name of the S3 bucket to send load balancer access logs. | `string` | `null` | no | | [node\_https\_ingress\_cidr\_blocks](#input\_node\_https\_ingress\_cidr\_blocks) | List of CIDR blocks to allow into the node over the HTTPs port | `list(string)` | 
[
  "10.0.0.0/8",
  "100.0.0.0/8"
]
| no | +| [node\_labels](#input\_node\_labels) | The labels to apply to the EKS nodes | `map(string)` | `{}` | no | | [node\_schedule\_shutdown\_hour](#input\_node\_schedule\_shutdown\_hour) | The hour of the day (0-23) the cluster should be shutdown. If left empty, the cluster will not be stopped. Will run every day otherwise. | `number` | `-1` | no | | [node\_schedule\_startup\_hour](#input\_node\_schedule\_startup\_hour) | The hour of the day (0-23) the cluster should be restarted. If left empty, the cluster will not be restarted after shutdown. Will run every weekday otherwise. | `number` | `-1` | no | | [node\_schedule\_timezone](#input\_node\_schedule\_timezone) | The timezone of the schedule. Ex: 'America/New\_York', 'America/Chicago', 'America/Denver', 'America/Los\_Angeles', 'Pacific/Honolulu' See: https://www.joda.org/joda-time/timezones.html | `string` | `"America/New_York"` | no | +| [node\_taints](#input\_node\_taints) | The taints to apply to the EKS nodes | `map(string)` | `{}` | no | +| [node\_volume\_delete\_on\_termination](#input\_node\_volume\_delete\_on\_termination) | Whether the volume should be deleted when the node is terminated. Defaults to true | `bool` | `true` | no | +| [node\_volume\_size](#input\_node\_volume\_size) | The size of the volume to use for the nodes. Defaults to 20 | `number` | `300` | no | +| [node\_volume\_type](#input\_node\_volume\_type) | The type of volume to use for the nodes. Defaults to gp2 | `string` | `"gp3"` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | OpenID Connect Audiences | `list(string)` | `[]` | no | | [private\_subnets](#input\_private\_subnets) | n/a | `list(any)` | n/a | yes | | [s3\_bucket\_access\_grants](#input\_s3\_bucket\_access\_grants) | A list of s3 bucket names to grant the cluster roles R/W access to | `list(string)` | `null` | no | @@ -196,6 +199,7 @@ Note that this example may create resources which cost money. Run `terraform des | [ssm\_tag\_patch\_group](#input\_ssm\_tag\_patch\_group) | SSM Patching group for instances. For more information: https://cloud.cms.gov/patching-prerequisites | `string` | `"AL2"` | no | | [ssm\_tag\_patch\_window](#input\_ssm\_tag\_patch\_window) | SSM Patching window for instances. For more information: https://cloud.cms.gov/patching-prerequisites | `string` | `"ITOPS-Wave1-Non-Mktplc-DevTestImpl-MW"` | no | | [tags](#input\_tags) | Global resource tags to apply to all resources | `map(any)` | `null` | no | +| [use\_bottlerocket](#input\_use\_bottlerocket) | Use Bottlerocket | `bool` | `false` | no | | [vpc\_cidr\_blocks](#input\_vpc\_cidr\_blocks) | List of VPC CIDR blocks | `list(string)` | n/a | yes | | [vpc\_id](#input\_vpc\_id) | n/a | `string` | n/a | yes | diff --git a/main.tf b/main.tf index 63f2653..7e4258f 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,7 @@ locals { - name = var.cluster_name - cluster_version = var.cluster_version - hoplimit_metadata = var.enable_hoplimit ? { http_put_response_hop_limit = 1 } : {} + name = var.cluster_name + cluster_version = var.cluster_version + hoplimit_metadata = var.enable_hoplimit ? { http_put_response_hop_limit = 1 } : {} } data "aws_ami" "eks_ami" { @@ -21,12 +21,6 @@ data "aws_security_groups" "delete_ebs_volumes_lambda_security_group" { # EKS Module ################################################################################ locals { - # Schedule config - create_schedule_startup = var.node_schedule_startup_hour >= 0 || var.node_schedule_startup_cron != "" - create_schedule_shutdown = var.node_schedule_shutdown_hour >= 0 || var.node_schedule_shutdown_cron != "" - create_schedule = local.create_schedule_startup || local.create_schedule_shutdown - node_schedule_shutdown_cron = var.node_schedule_shutdown_cron != "" ? var.node_schedule_shutdown_cron : "0 ${var.node_schedule_shutdown_hour} * * *" - node_schedule_startup_cron = var.node_schedule_startup_cron != "" ? var.node_schedule_startup_cron : "0 ${var.node_schedule_startup_hour} * * 1-5" instance_policy_tags = var.enable_ssm_patching ? { "Patch Group" = var.ssm_tag_patch_group, "Patch Window" = var.ssm_tag_patch_window } : {} instance_tags = merge(local.instance_policy_tags, var.instance_tags) diff --git a/variables.tf b/variables.tf index c63eba3..bbcb323 100644 --- a/variables.tf +++ b/variables.tf @@ -20,10 +20,10 @@ variable "general_node_pool" { default = { instance_type = "c5.2xlarge" # ami_type = "BOTTLEROCKET_x86_64" - desired_size = 3 - max_size = 5 - min_size = 2 - use_custom_launch_template = false + desired_size = 3 + max_size = 5 + min_size = 2 + use_custom_launch_template = false # Map of label flags for kubelets. labels = { general = "true" } taints = {} @@ -273,30 +273,12 @@ variable "create_cosign_iam_role" { type = bool } -variable "autoscaling_group_tags" { - description = "Tags to apply to all autoscaling groups created" - default = {} - type = map(any) -} - variable "ami_regex_override" { description = "Overrides default AMI lookup regex, which grabs latest AMI matching cluster_version by default" default = "" type = string } -variable "node_schedule_shutdown_cron" { - type = string - default = "" - description = "The cron schedule for the cluster to be shutdown. If left empty, the cluster will not be stopped. Will run every day otherwise." -} - -variable "node_schedule_startup_cron" { - type = string - default = "" - description = "The cron schedule for the cluster to be restarted. If left empty, the cluster will not be restarted after shutdown. Will run every weekday otherwise." -} - variable "node_schedule_shutdown_hour" { type = number default = -1 @@ -347,11 +329,6 @@ variable "federated_access_role" { } -variable "enable_self_managed_nodes" { - type = bool - default = true - description = "Enables self managed nodes" -} variable "force_update_version" { type = bool From 843d564406e637efbaaba28173ff3c06ed26e75f Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Mon, 25 Mar 2024 11:57:16 -0400 Subject: [PATCH 20/33] update for recommendations --- main.tf | 53 +++++++++++++++++---------------- templates/bottlerocket.toml.tpl | 4 +-- variables.tf | 23 ++++++++------ 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/main.tf b/main.tf index 63f2653..adbf94b 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,7 @@ locals { - name = var.cluster_name - cluster_version = var.cluster_version - hoplimit_metadata = var.enable_hoplimit ? { http_put_response_hop_limit = 1 } : {} + name = var.cluster_name + cluster_version = var.cluster_version + hoplimit_metadata = var.enable_hoplimit ? { http_put_response_hop_limit = 1 } : {} } data "aws_ami" "eks_ami" { @@ -51,27 +51,6 @@ locals { # EKS Fully managed nodes ################################################################################ locals { - base_block_device_mappings = [ - { - device_name = "/dev/xvda" - ebs = { - volume_size = "8" - volume_type = "gp3" - delete_on_termination = true - encrypted = true - } - }, - { - device_name = "/dev/xvdb" - ebs = { - volume_size = var.node_volume_size - volume_type = var.node_volume_type - delete_on_termination = var.node_volume_delete_on_termination - encrypted = true - } - } - ] - node_labels = merge( var.node_labels ) @@ -80,6 +59,7 @@ locals { cluster_name = var.cluster_name cluster_endpoint = module.eks.cluster_endpoint cluster_ca_data = module.eks.cluster_certificate_authority_data + pod_pids_limit = var.bottlerocket_pod_pids_limit max_namespaces = 10000 node_labels = join("\n", [for label, value in local.node_labels : "\"${label}\" = \"${value}\""]) node_taints = join("\n", [for taint, value in var.node_taints : "\"${taint}\" = \"${value}\""]) @@ -109,10 +89,31 @@ locals { desired_size = v.desired_size # This is dynamically creating the block device mappings based on the AMI type - block_device_mappings = var.use_bottlerocket ? local.base_block_device_mappings : [ + block_device_mappings = var.use_bottlerocket ? [ { device_name = "/dev/xvda" - ebs = local.base_block_device_mappings[1].ebs + ebs = { + volume_size = 8 + volume_type = "gp3" + delete_on_termination = true + } + }, + { + device_name = "dev/xvdb" + ebs = { + volume_size = try(v.volume_size, "300") + volume_type = try(v.volume_type, "gp3") + delete_on_termination = try(v.delete_on_termination, true) + } + } + ] : [ + { + device_name = "/dev/xvda" + ebs = { + volume_size = try(v.volume_size, "300") + volume_type = try(v.volume_type, "gp3") + delete_on_termination = try(v.delete_on_termination, true) + } } ] diff --git a/templates/bottlerocket.toml.tpl b/templates/bottlerocket.toml.tpl index 94b9c62..a41c362 100644 --- a/templates/bottlerocket.toml.tpl +++ b/templates/bottlerocket.toml.tpl @@ -1,12 +1,12 @@ # settings.kubernetes section from bootstrap_extra_args in default template -pod-pids-limit = 1000 +pod-pids-limit = ${pod_pids_limit} # Set autoscaling wait [settings.autoscaling] should-wait = true # The admin host container provides SSH access and runs with "superpowers". -# It is disabled by default, but can be disabled explicitly. +# It is disabled by default, but can be enabled explicitly. [settings.host-containers.admin] enabled = false diff --git a/variables.tf b/variables.tf index c63eba3..5207ed4 100644 --- a/variables.tf +++ b/variables.tf @@ -18,12 +18,11 @@ variable "general_node_pool" { type = any description = "General node pool, required for hosting core services" default = { - instance_type = "c5.2xlarge" - # ami_type = "BOTTLEROCKET_x86_64" - desired_size = 3 - max_size = 5 - min_size = 2 - use_custom_launch_template = false + instance_type = "c5.2xlarge" + desired_size = 3 + max_size = 5 + min_size = 2 + use_custom_launch_template = false # Map of label flags for kubelets. labels = { general = "true" } taints = {} @@ -116,19 +115,19 @@ variable "node_https_ingress_cidr_blocks" { type = list(string) } -variable "node_volume_size" { +variable "node_default_volume_size" { description = "The size of the volume to use for the nodes. Defaults to 20" default = 300 type = number } -variable "node_volume_type" { +variable "node_default_volume_type" { description = "The type of volume to use for the nodes. Defaults to gp2" default = "gp3" type = string } -variable "node_volume_delete_on_termination" { +variable "node_default_volume_delete_on_termination" { description = "Whether the volume should be deleted when the node is terminated. Defaults to true" default = true type = bool @@ -412,3 +411,9 @@ variable "node_taints" { type = map(string) default = {} } + +variable "bottlerocket_pod_pids_limit" { + type = number + default = 1000 + description = "The maximum number of processes that can be created in a pod" +} From 1807177cff275d5dd2d45b9ad1ff8aa6265b948c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:59:27 +0000 Subject: [PATCH 21/33] [pre-commit.ci lite] apply automatic fixes --- README.md | 4 +--- variables.tf | 41 ----------------------------------------- 2 files changed, 1 insertion(+), 44 deletions(-) diff --git a/README.md b/README.md index 456e864..0ae36c1 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ Note that this example may create resources which cost money. Run `terraform des | [ami\_date](#input\_ami\_date) | n/a | `string` | `""` | no | | [ami\_owner\_override](#input\_ami\_owner\_override) | AMI owner override | `list(string)` | 
[
  ""
]
| no | | [ami\_regex\_override](#input\_ami\_regex\_override) | Overrides default AMI lookup regex, which grabs latest AMI matching cluster\_version by default | `string` | `""` | no | +| [bottlerocket\_pod\_pids\_limit](#input\_bottlerocket\_pod\_pids\_limit) | The maximum number of processes that can be created in a pod | `number` | `1000` | no | | [cluster\_additional\_sg\_prefix\_lists](#input\_cluster\_additional\_sg\_prefix\_lists) | n/a | `list(string)` | n/a | yes | | [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logging to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | 
[
  "api",
  "audit",
  "authenticator",
  "controllerManager",
  "scheduler"
]
| no | | [cluster\_name](#input\_cluster\_name) | n/a | `string` | n/a | yes | @@ -189,9 +190,6 @@ Note that this example may create resources which cost money. Run `terraform des | [node\_schedule\_startup\_hour](#input\_node\_schedule\_startup\_hour) | The hour of the day (0-23) the cluster should be restarted. If left empty, the cluster will not be restarted after shutdown. Will run every weekday otherwise. | `number` | `-1` | no | | [node\_schedule\_timezone](#input\_node\_schedule\_timezone) | The timezone of the schedule. Ex: 'America/New\_York', 'America/Chicago', 'America/Denver', 'America/Los\_Angeles', 'Pacific/Honolulu' See: https://www.joda.org/joda-time/timezones.html | `string` | `"America/New_York"` | no | | [node\_taints](#input\_node\_taints) | The taints to apply to the EKS nodes | `map(string)` | `{}` | no | -| [node\_volume\_delete\_on\_termination](#input\_node\_volume\_delete\_on\_termination) | Whether the volume should be deleted when the node is terminated. Defaults to true | `bool` | `true` | no | -| [node\_volume\_size](#input\_node\_volume\_size) | The size of the volume to use for the nodes. Defaults to 20 | `number` | `300` | no | -| [node\_volume\_type](#input\_node\_volume\_type) | The type of volume to use for the nodes. Defaults to gp2 | `string` | `"gp3"` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | OpenID Connect Audiences | `list(string)` | `[]` | no | | [private\_subnets](#input\_private\_subnets) | n/a | `list(any)` | n/a | yes | | [s3\_bucket\_access\_grants](#input\_s3\_bucket\_access\_grants) | A list of s3 bucket names to grant the cluster roles R/W access to | `list(string)` | `null` | no | diff --git a/variables.tf b/variables.tf index 5207ed4..4bc767c 100644 --- a/variables.tf +++ b/variables.tf @@ -115,24 +115,6 @@ variable "node_https_ingress_cidr_blocks" { type = list(string) } -variable "node_default_volume_size" { - description = "The size of the volume to use for the nodes. Defaults to 20" - default = 300 - type = number -} - -variable "node_default_volume_type" { - description = "The type of volume to use for the nodes. Defaults to gp2" - default = "gp3" - type = string -} - -variable "node_default_volume_delete_on_termination" { - description = "Whether the volume should be deleted when the node is terminated. Defaults to true" - default = true - type = bool -} - variable "alb_restricted_hosts" { type = set(string) description = "A list of allowable host for private alb" @@ -272,30 +254,12 @@ variable "create_cosign_iam_role" { type = bool } -variable "autoscaling_group_tags" { - description = "Tags to apply to all autoscaling groups created" - default = {} - type = map(any) -} - variable "ami_regex_override" { description = "Overrides default AMI lookup regex, which grabs latest AMI matching cluster_version by default" default = "" type = string } -variable "node_schedule_shutdown_cron" { - type = string - default = "" - description = "The cron schedule for the cluster to be shutdown. If left empty, the cluster will not be stopped. Will run every day otherwise." -} - -variable "node_schedule_startup_cron" { - type = string - default = "" - description = "The cron schedule for the cluster to be restarted. If left empty, the cluster will not be restarted after shutdown. Will run every weekday otherwise." -} - variable "node_schedule_shutdown_hour" { type = number default = -1 @@ -346,11 +310,6 @@ variable "federated_access_role" { } -variable "enable_self_managed_nodes" { - type = bool - default = true - description = "Enables self managed nodes" -} variable "force_update_version" { type = bool From f07afe267b469a59a3d55e64f80f995e3bd3e4a2 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Mon, 25 Mar 2024 12:01:55 -0400 Subject: [PATCH 22/33] add volume encryption --- main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.tf b/main.tf index 8d760ce..9ffc046 100644 --- a/main.tf +++ b/main.tf @@ -90,6 +90,7 @@ locals { volume_size = 8 volume_type = "gp3" delete_on_termination = true + encrypted = true } }, { @@ -98,6 +99,7 @@ locals { volume_size = try(v.volume_size, "300") volume_type = try(v.volume_type, "gp3") delete_on_termination = try(v.delete_on_termination, true) + encrypted = true } } ] : [ @@ -107,6 +109,7 @@ locals { volume_size = try(v.volume_size, "300") volume_type = try(v.volume_type, "gp3") delete_on_termination = try(v.delete_on_termination, true) + encrypted = true } } ] From ba9e78f316f16f7274a74864c53c9306545cf436 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Mon, 25 Mar 2024 12:04:15 -0400 Subject: [PATCH 23/33] fix vol name --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 9ffc046..cceb034 100644 --- a/main.tf +++ b/main.tf @@ -94,7 +94,7 @@ locals { } }, { - device_name = "dev/xvdb" + device_name = "/dev/xvdb" ebs = { volume_size = try(v.volume_size, "300") volume_type = try(v.volume_type, "gp3") From 56ad36f3fd63a5356d3b5b055486fd50f79c1902 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Tue, 26 Mar 2024 14:08:16 -0400 Subject: [PATCH 24/33] updates for PR --- main.tf | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/main.tf b/main.tf index cceb034..4156b61 100644 --- a/main.tf +++ b/main.tf @@ -4,6 +4,8 @@ locals { hoplimit_metadata = var.enable_hoplimit ? { http_put_response_hop_limit = 1 } : {} } +# If an AMI owner is specified, use that owner. Otherwise, use the CMS AMI by default, +# with the capability to pass 'use_bottlerocket' var to use the bottlerocket AMI. data "aws_ami" "eks_ami" { most_recent = true name_regex = var.use_bottlerocket ? "^bottlerocket-aws-k8s-${var.cluster_version}-x86_64-v1.17.0" : (var.ami_regex_override == "" ? "^amzn2-eks-${var.cluster_version}-gi-${var.ami_date}" : var.ami_regex_override) @@ -45,17 +47,13 @@ locals { # EKS Fully managed nodes ################################################################################ locals { - node_labels = merge( - var.node_labels - ) - bottlerocket_bootstrap_template = templatefile("${path.module}/templates/bottlerocket.toml.tpl", { cluster_name = var.cluster_name cluster_endpoint = module.eks.cluster_endpoint cluster_ca_data = module.eks.cluster_certificate_authority_data pod_pids_limit = var.bottlerocket_pod_pids_limit max_namespaces = 10000 - node_labels = join("\n", [for label, value in local.node_labels : "\"${label}\" = \"${value}\""]) + node_labels = join("\n", [for label, value in var.node_labels : "\"${label}\" = \"${value}\""]) node_taints = join("\n", [for taint, value in var.node_taints : "\"${taint}\" = \"${value}\""]) }) From d51cc17d580632c5d744ac06d171961b7efef1f8 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Tue, 26 Mar 2024 14:59:49 -0400 Subject: [PATCH 25/33] update for module cleanup --- main.tf | 51 ++++++++++++++++++++++++++------------------------- variables.tf | 14 +------------- 2 files changed, 27 insertions(+), 38 deletions(-) diff --git a/main.tf b/main.tf index 4156b61..55cbb5c 100644 --- a/main.tf +++ b/main.tf @@ -47,15 +47,21 @@ locals { # EKS Fully managed nodes ################################################################################ locals { - bottlerocket_bootstrap_template = templatefile("${path.module}/templates/bottlerocket.toml.tpl", { - cluster_name = var.cluster_name - cluster_endpoint = module.eks.cluster_endpoint - cluster_ca_data = module.eks.cluster_certificate_authority_data - pod_pids_limit = var.bottlerocket_pod_pids_limit - max_namespaces = 10000 - node_labels = join("\n", [for label, value in var.node_labels : "\"${label}\" = \"${value}\""]) - node_taints = join("\n", [for taint, value in var.node_taints : "\"${taint}\" = \"${value}\""]) - }) + shared_node_labels = { + for k, v in var.custom_node_pools : k => { + for label_key, label_value in try(v.labels, {}) : label_key => label_value + } + } + + shared_node_taints = { + for k, v in var.custom_node_pools : k => [ + for taint_key, taint_string in try(v.taints, {}) : { + key = taint_key + value = element(split(":", taint_string), 0) + effect = "NO_SCHEDULE" + } + ] + } eks_node_pools = { for k, v in merge({ general = var.general_node_pool }, var.custom_node_pools) : k => { group_name = k @@ -72,7 +78,15 @@ locals { use_custom_launch_template = try(v.use_custom_launch_template, true) ami_type = var.use_bottlerocket ? "BOTTLEROCKET_x86_64" : "AL2_x86_64" platform = var.use_bottlerocket ? "bottlerocket" : "linux" - bootstrap_extra_args = local.bottlerocket_bootstrap_template + bootstrap_extra_args = templatefile("${path.module}/templates/bottlerocket.toml.tpl", { + cluster_name = var.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_ca_data = module.eks.cluster_certificate_authority_data + pod_pids_limit = var.bottlerocket_pod_pids_limit + max_namespaces = 10000 + node_labels = local.shared_node_labels[k] + node_taints = local.shared_node_taints[k] + }) subnet_ids = coalescelist(try(v.subnet_ids, []), var.host_subnets, var.private_subnets) @@ -123,19 +137,6 @@ locals { tags = merge(var.tags, local.instance_tags, try(v.tags, null)) - taints = [ - for taint_key, taint_string in try(v.taints, {}) : { - key = taint_key - value = element(split(":", taint_string), 0) - effect = "NO_SCHEDULE" - } - ] - - labels = { - for label_key, label_value in try(v.labels, {}) : - label_key => label_value - } - create_schedule = var.node_schedule_shutdown_hour >= 0 || var.node_schedule_startup_hour >= 0 schedules = merge( var.node_schedule_shutdown_hour < 0 ? {} : { @@ -233,8 +234,8 @@ module "eks_managed_node_groups" { ami_type = each.value.ami_type metadata_options = each.value.metadata_options tags = each.value.tags - taints = each.value.taints - labels = each.value.labels + taints = local.shared_node_taints[each.key] + labels = local.shared_node_labels[each.key] create_schedule = each.value.create_schedule schedules = each.value.schedules force_update_version = var.force_update_version diff --git a/variables.tf b/variables.tf index 4bc767c..d103482 100644 --- a/variables.tf +++ b/variables.tf @@ -350,7 +350,7 @@ variable "ssm_tag_patch_window" { variable "ami_owner_override" { type = list(string) default = [""] - description = "AMI owner override" + description = "Override the AWS Account owner used to look up AMI's for the cluster nodes" } variable "use_bottlerocket" { @@ -359,18 +359,6 @@ variable "use_bottlerocket" { description = "Use Bottlerocket" } -variable "node_labels" { - description = "The labels to apply to the EKS nodes" - type = map(string) - default = {} -} - -variable "node_taints" { - description = "The taints to apply to the EKS nodes" - type = map(string) - default = {} -} - variable "bottlerocket_pod_pids_limit" { type = number default = 1000 From b4afced20c97cef162248b27bbbab780dbc6b710 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:00:33 +0000 Subject: [PATCH 26/33] [pre-commit.ci lite] apply automatic fixes --- README.md | 4 +--- main.tf | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0ae36c1..a3c9c26 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ Note that this example may create resources which cost money. Run `terraform des | [alb\_ssl\_security\_policy](#input\_alb\_ssl\_security\_policy) | ALB SSL Security Policy | `string` | `"ELBSecurityPolicy-TLS13-1-2-Res-2021-06"` | no | | [alb\_subnets\_by\_zone](#input\_alb\_subnets\_by\_zone) | n/a | `map(string)` | n/a | yes | | [ami\_date](#input\_ami\_date) | n/a | `string` | `""` | no | -| [ami\_owner\_override](#input\_ami\_owner\_override) | AMI owner override | `list(string)` | 
[
  ""
]
| no | +| [ami\_owner\_override](#input\_ami\_owner\_override) | Override the AWS Account owner used to look up AMI's for the cluster nodes | `list(string)` | 
[
  ""
]
| no | | [ami\_regex\_override](#input\_ami\_regex\_override) | Overrides default AMI lookup regex, which grabs latest AMI matching cluster\_version by default | `string` | `""` | no | | [bottlerocket\_pod\_pids\_limit](#input\_bottlerocket\_pod\_pids\_limit) | The maximum number of processes that can be created in a pod | `number` | `1000` | no | | [cluster\_additional\_sg\_prefix\_lists](#input\_cluster\_additional\_sg\_prefix\_lists) | n/a | `list(string)` | n/a | yes | @@ -185,11 +185,9 @@ Note that this example may create resources which cost money. Run `terraform des | [instance\_tags](#input\_instance\_tags) | Instance custom tags | `map(any)` | `null` | no | | [logging\_bucket](#input\_logging\_bucket) | Name of the S3 bucket to send load balancer access logs. | `string` | `null` | no | | [node\_https\_ingress\_cidr\_blocks](#input\_node\_https\_ingress\_cidr\_blocks) | List of CIDR blocks to allow into the node over the HTTPs port | `list(string)` | 
[
  "10.0.0.0/8",
  "100.0.0.0/8"
]
| no | -| [node\_labels](#input\_node\_labels) | The labels to apply to the EKS nodes | `map(string)` | `{}` | no | | [node\_schedule\_shutdown\_hour](#input\_node\_schedule\_shutdown\_hour) | The hour of the day (0-23) the cluster should be shutdown. If left empty, the cluster will not be stopped. Will run every day otherwise. | `number` | `-1` | no | | [node\_schedule\_startup\_hour](#input\_node\_schedule\_startup\_hour) | The hour of the day (0-23) the cluster should be restarted. If left empty, the cluster will not be restarted after shutdown. Will run every weekday otherwise. | `number` | `-1` | no | | [node\_schedule\_timezone](#input\_node\_schedule\_timezone) | The timezone of the schedule. Ex: 'America/New\_York', 'America/Chicago', 'America/Denver', 'America/Los\_Angeles', 'Pacific/Honolulu' See: https://www.joda.org/joda-time/timezones.html | `string` | `"America/New_York"` | no | -| [node\_taints](#input\_node\_taints) | The taints to apply to the EKS nodes | `map(string)` | `{}` | no | | [openid\_connect\_audiences](#input\_openid\_connect\_audiences) | OpenID Connect Audiences | `list(string)` | `[]` | no | | [private\_subnets](#input\_private\_subnets) | n/a | `list(any)` | n/a | yes | | [s3\_bucket\_access\_grants](#input\_s3\_bucket\_access\_grants) | A list of s3 bucket names to grant the cluster roles R/W access to | `list(string)` | `null` | no | diff --git a/main.tf b/main.tf index 55cbb5c..ad865cf 100644 --- a/main.tf +++ b/main.tf @@ -78,7 +78,7 @@ locals { use_custom_launch_template = try(v.use_custom_launch_template, true) ami_type = var.use_bottlerocket ? "BOTTLEROCKET_x86_64" : "AL2_x86_64" platform = var.use_bottlerocket ? "bottlerocket" : "linux" - bootstrap_extra_args = templatefile("${path.module}/templates/bottlerocket.toml.tpl", { + bootstrap_extra_args = templatefile("${path.module}/templates/bottlerocket.toml.tpl", { cluster_name = var.cluster_name cluster_endpoint = module.eks.cluster_endpoint cluster_ca_data = module.eks.cluster_certificate_authority_data From f13d39984027280a0f31906fdbd781441b363716 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Tue, 26 Mar 2024 15:04:29 -0400 Subject: [PATCH 27/33] update to make loop consistent --- main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 55cbb5c..6dc05e0 100644 --- a/main.tf +++ b/main.tf @@ -48,13 +48,13 @@ locals { ################################################################################ locals { shared_node_labels = { - for k, v in var.custom_node_pools : k => { + for k, v in merge({ general = var.general_node_pool }, var.custom_node_pools) : k => { for label_key, label_value in try(v.labels, {}) : label_key => label_value } } shared_node_taints = { - for k, v in var.custom_node_pools : k => [ + for k, v in merge({ general = var.general_node_pool }, var.custom_node_pools) : k => [ for taint_key, taint_string in try(v.taints, {}) : { key = taint_key value = element(split(":", taint_string), 0) From 02536fea1989e8c6bbeb4d8f623417fb9cf193df Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Tue, 26 Mar 2024 15:08:09 -0400 Subject: [PATCH 28/33] wrap var in string for interpolation --- templates/bottlerocket.toml.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/bottlerocket.toml.tpl b/templates/bottlerocket.toml.tpl index a41c362..1aaf25f 100644 --- a/templates/bottlerocket.toml.tpl +++ b/templates/bottlerocket.toml.tpl @@ -24,7 +24,7 @@ lockdown = "integrity" "user.max_user_namespaces" = "${max_namespaces}" [settings.kubernetes.node-labels] -${node_labels} +"${node_labels}" [settings.kubernetes.node-taints] -${node_taints} +"${node_taints}" From 80a2ce8f188b5a4a6164e37080b08b12a81ff781 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Tue, 26 Mar 2024 15:14:00 -0400 Subject: [PATCH 29/33] update toml to provide k-v pairs instead of string of map --- templates/bottlerocket.toml.tpl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/templates/bottlerocket.toml.tpl b/templates/bottlerocket.toml.tpl index 1aaf25f..29d7f64 100644 --- a/templates/bottlerocket.toml.tpl +++ b/templates/bottlerocket.toml.tpl @@ -24,7 +24,11 @@ lockdown = "integrity" "user.max_user_namespaces" = "${max_namespaces}" [settings.kubernetes.node-labels] -"${node_labels}" +%{ for label_key, label_value in node_labels ~} +"${label_key}" = "${label_value}" +%{ endfor ~} [settings.kubernetes.node-taints] -"${node_taints}" +%{ for taint in node_taints ~} +"${taint.key}" = "${taint.value}:${taint.effect}" +%{ endfor ~} From c0c04cd469d9a97d0e91f6c49e39c16f4d7920b6 Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Tue, 26 Mar 2024 16:12:37 -0400 Subject: [PATCH 30/33] add if statements just in case --- templates/bottlerocket.toml.tpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/templates/bottlerocket.toml.tpl b/templates/bottlerocket.toml.tpl index 29d7f64..8a3f63f 100644 --- a/templates/bottlerocket.toml.tpl +++ b/templates/bottlerocket.toml.tpl @@ -23,12 +23,16 @@ lockdown = "integrity" [settings.kernel.sysctl] "user.max_user_namespaces" = "${max_namespaces}" +%{ if length(node_labels) > 0 ~} [settings.kubernetes.node-labels] %{ for label_key, label_value in node_labels ~} "${label_key}" = "${label_value}" %{ endfor ~} +%{ endif ~} +%{ if length(node_taints) > 0 ~} [settings.kubernetes.node-taints] %{ for taint in node_taints ~} "${taint.key}" = "${taint.value}:${taint.effect}" %{ endfor ~} +%{ endif ~} From 2a52384add9e84f4332525f1ccab182cc6342b70 Mon Sep 17 00:00:00 2001 From: dahoward5 <161613558+dahoward5@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:19:05 -0400 Subject: [PATCH 31/33] update ami owner logic Co-authored-by: Charles Bushong --- main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index a5d5bb6..3f930b6 100644 --- a/main.tf +++ b/main.tf @@ -9,7 +9,8 @@ locals { data "aws_ami" "eks_ami" { most_recent = true name_regex = var.use_bottlerocket ? "^bottlerocket-aws-k8s-${var.cluster_version}-x86_64-v1.17.0" : (var.ami_regex_override == "" ? "^amzn2-eks-${var.cluster_version}-gi-${var.ami_date}" : var.ami_regex_override) - owners = var.use_bottlerocket ? ["092701018921"] : (length(var.ami_owner_override) > 0 && var.ami_owner_override[0] != "" ? var.ami_owner_override : ["743302140042"]) + # If an ami_owner_override is provided, use it. Otherwise use the AWS AMI's for bottlerocket, and CMS AMIs AL2 + owners = var.ami_owner_override != "" ? [var.ami_owner_override] : (var.use_bottlerocket ? ["092701018921"] : ["743302140042"]) } data "aws_security_groups" "delete_ebs_volumes_lambda_security_group" { From 5e1384e332f63dec45c38acf041a761ce353468d Mon Sep 17 00:00:00 2001 From: dahoward5 Date: Tue, 26 Mar 2024 16:20:10 -0400 Subject: [PATCH 32/33] update ami override var --- variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/variables.tf b/variables.tf index d103482..8017967 100644 --- a/variables.tf +++ b/variables.tf @@ -348,8 +348,8 @@ variable "ssm_tag_patch_window" { } variable "ami_owner_override" { - type = list(string) - default = [""] + type = string + default = "" description = "Override the AWS Account owner used to look up AMI's for the cluster nodes" } From d2e39b27d5be6059e0ca9d677526e9156a7511aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 20:21:36 +0000 Subject: [PATCH 33/33] [pre-commit.ci lite] apply automatic fixes --- README.md | 2 +- main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a3c9c26..afc468e 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ Note that this example may create resources which cost money. Run `terraform des | [alb\_ssl\_security\_policy](#input\_alb\_ssl\_security\_policy) | ALB SSL Security Policy | `string` | `"ELBSecurityPolicy-TLS13-1-2-Res-2021-06"` | no | | [alb\_subnets\_by\_zone](#input\_alb\_subnets\_by\_zone) | n/a | `map(string)` | n/a | yes | | [ami\_date](#input\_ami\_date) | n/a | `string` | `""` | no | -| [ami\_owner\_override](#input\_ami\_owner\_override) | Override the AWS Account owner used to look up AMI's for the cluster nodes | `list(string)` | 
[
  ""
]
| no | +| [ami\_owner\_override](#input\_ami\_owner\_override) | Override the AWS Account owner used to look up AMI's for the cluster nodes | `string` | `""` | no | | [ami\_regex\_override](#input\_ami\_regex\_override) | Overrides default AMI lookup regex, which grabs latest AMI matching cluster\_version by default | `string` | `""` | no | | [bottlerocket\_pod\_pids\_limit](#input\_bottlerocket\_pod\_pids\_limit) | The maximum number of processes that can be created in a pod | `number` | `1000` | no | | [cluster\_additional\_sg\_prefix\_lists](#input\_cluster\_additional\_sg\_prefix\_lists) | n/a | `list(string)` | n/a | yes | diff --git a/main.tf b/main.tf index 3f930b6..47ef7fe 100644 --- a/main.tf +++ b/main.tf @@ -10,7 +10,7 @@ data "aws_ami" "eks_ami" { most_recent = true name_regex = var.use_bottlerocket ? "^bottlerocket-aws-k8s-${var.cluster_version}-x86_64-v1.17.0" : (var.ami_regex_override == "" ? "^amzn2-eks-${var.cluster_version}-gi-${var.ami_date}" : var.ami_regex_override) # If an ami_owner_override is provided, use it. Otherwise use the AWS AMI's for bottlerocket, and CMS AMIs AL2 - owners = var.ami_owner_override != "" ? [var.ami_owner_override] : (var.use_bottlerocket ? ["092701018921"] : ["743302140042"]) + owners = var.ami_owner_override != "" ? [var.ami_owner_override] : (var.use_bottlerocket ? ["092701018921"] : ["743302140042"]) } data "aws_security_groups" "delete_ebs_volumes_lambda_security_group" {