feat: add nightly testing for rke2 (#808)

## Description
This pull request introduces a new nightly test of uds-core on RKE2. The
pipeline runs in parallel alongside our existing EKS nightly tests.
Included in this PR are additional IaC resources to deploy RKE2, an
additional bundle for RKE2 and updated task files/workflows.

Details:
- Changes `test-infra` directory layout to support multiple k8s
distributions (merges `test-infra/buckets-iac` and `test-infra/rds-iac`
into `test-infra/aws/eks` and adds `test-infra/aws/rke2`)
- Adds IaC for a minimal deployment of RKE2 under `test-infra/aws/rke2`
- Introduces new test bundle for uds-core on RKE2
(`.github/bundles/rke2`), changes `.github/bundles` directory layout to
support additional distros (moves `.github/bundles/*.yaml` to
`.github/bundles/eks`)
- Introduces new workflow for nightly tests of uds-core on rke2
(`.github/workflows/test-rke2.yaml`)
- Updates workflows and task files to support testing across multiple
K8s distros.

Callouts:
- uses existing vpc and subnets in ci account
- the `rke2_version` var will be used to query the latest ami for the
given minor version
- uses rhel amis built from uds-rke2-image-builder repo
- includes support for irsa
- no cluster-autoscaler deployed, currently hardcoded to provision 4
`m5.2xlarge` nodes
- does not deploy any external dependencies apart from s3 buckets for
loki and velero

Example pipeline run with new RKE2 tests can be seen
[here](https://github.com/defenseunicorns/uds-core/actions/runs/11150004170/).

## Related Issue

Fixes #726 

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [ ] Test, docs, adr added or updated as needed
- [ ] [Contributor
Guide](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md)
followed

.github/bundles/uds-bundle.yaml → .github/bundles/eks/uds-bundle.yaml

@@ -12,7 +12,7 @@ packages:
     ref: v0.41.0
 
   - name: core
-    path: ../../build/
+    path: ../../../build
     # x-release-please-start-version
     ref: 0.28.0
     # x-release-please-end
@@ -25,7 +25,7 @@ packages:
             - name: VELERO_USE_SECRET
               description: "Toggle use secret off to use IRSA."
               path: credentials.useSecret
-            - name: VELERO_IRSA_ANNOTATION
+            - name: VELERO_IRSA_ROLE_ARN
               description: "IRSA ARN annotation to use for Velero"
               path: serviceAccount.server.annotations.eks\.amazonaws\.com/role-arn
       loki:
@@ -50,7 +50,7 @@ packages:
             - name: LOKI_S3_REGION
               description: "The S3 region"
               path: loki.storage.s3.region
-            - name: LOKI_IRSA_ANNOTATION
+            - name: LOKI_IRSA_ROLE_ARN
               description: "The irsa role annotation"
               path: serviceAccount.annotations.eks\.amazonaws\.com/role-arn
       grafana:

.github/bundles/uds-config.yaml → .github/bundles/eks/uds-config.yaml

@@ -8,9 +8,9 @@ variables:
     loki_ruler_bucket: ${ZARF_VAR_LOKI_S3_BUCKET}
     loki_admin_bucket: ${ZARF_VAR_LOKI_S3_BUCKET}
     loki_s3_region: ${ZARF_VAR_LOKI_S3_AWS_REGION}
-    loki_irsa_annotation: ${ZARF_VAR_LOKI_S3_ROLE_ARN}
+    loki_irsa_role_arn: ${ZARF_VAR_LOKI_S3_ROLE_ARN}
     velero_use_secret: false
-    velero_irsa_annotation: "${ZARF_VAR_VELERO_S3_ROLE_ARN}"
+    velero_irsa_role_arn: "${ZARF_VAR_VELERO_S3_ROLE_ARN}"
     velero_bucket: ${ZARF_VAR_VELERO_S3_BUCKET}
     velero_bucket_region: ${ZARF_VAR_VELERO_S3_AWS_REGION}
     velero_bucket_provider_url: ""

.github/bundles/rke2/uds-bundle.yaml

@@ -0,0 +1,78 @@
+kind: UDSBundle
+metadata:
+  name: uds-core-rke2-nightly
+  description: A UDS bundle for deploying RKE2 and UDS Core
+  # x-release-please-start-version
+  version: "0.28.0"
+  # x-release-please-end
+
+packages:
+  - name: pod-identity-webhook
+    repository: ghcr.io/defenseunicorns/packages/uds/pod-identity-webhook
+    ref: 0.3.1-upstream
+
+  - name: init
+    repository: ghcr.io/zarf-dev/packages/init
+    ref: v0.40.1
+    overrides:
+      zarf-registry:
+        docker-registry:
+          variables:
+            - path: affinity.custom
+              name: REGISTRY_AFFINITY_CUSTOM_UDS
+            - path: persistence.accessMode
+              name: REGISTRY_PVC_ACCESS_MODE
+              default: ReadWriteMany
+      zarf-seed-registry:
+        docker-registry:
+          variables:
+            - path: affinity.custom
+              name: REGISTRY_AFFINITY_CUSTOM_UDS
+            - path: persistence.accessMode
+              name: REGISTRY_PVC_ACCESS_MODE
+              default: ReadWriteMany
+
+  - name: core
+    path: ../../../build
+    # x-release-please-start-version
+    ref: 0.28.0
+    # x-release-please-end
+    optionalComponents:
+      - metrics-server
+    overrides:
+      velero:
+        velero:
+          variables:
+            - name: VELERO_USE_SECRET
+              description: "Toggle use secret off to use IRSA."
+              path: credentials.useSecret
+            - name: VELERO_IRSA_ROLE_ARN
+              description: "IRSA ARN annotation to use for Velero"
+              path: serviceAccount.server.annotations.irsa/role-arn
+      loki:
+        loki:
+          values:
+            - path: loki.storage.s3.endpoint
+              value: ""
+            - path: loki.storage.s3.secretAccessKey
+              value: ""
+            - path: loki.storage.s3.accessKeyId
+              value: ""
+            - path: global.dnsService
+              value: rke2-coredns-rke2-coredns
+          variables:
+            - name: LOKI_CHUNKS_BUCKET
+              description: "The object storage bucket for Loki chunks"
+              path: loki.storage.bucketNames.chunks
+            - name: LOKI_RULER_BUCKET
+              description: "The object storage bucket for Loki ruler"
+              path: loki.storage.bucketNames.ruler
+            - name: LOKI_ADMIN_BUCKET
+              description: "The object storage bucket for Loki admin"
+              path: loki.storage.bucketNames.admin
+            - name: LOKI_S3_REGION
+              description: "The S3 region"
+              path: loki.storage.s3.region
+            - name: LOKI_IRSA_ROLE_ARN
+              description: "The irsa role annotation"
+              path: serviceAccount.annotations.irsa/role-arn

.github/bundles/rke2/uds-config.yaml

@@ -0,0 +1,18 @@
+# Overwritten by ci-iac-aws package
+options:
+  architecture: amd64
+
+variables:
+  core:
+    loki_chunks_bucket: ${ZARF_VAR_LOKI_S3_BUCKET}
+    loki_ruler_bucket: ${ZARF_VAR_LOKI_S3_BUCKET}
+    loki_admin_bucket: ${ZARF_VAR_LOKI_S3_BUCKET}
+    loki_s3_region: ${ZARF_VAR_LOKI_S3_AWS_REGION}
+    loki_irsa_role_arn: ${ZARF_VAR_LOKI_S3_ROLE_ARN}
+    velero_use_secret: false
+    velero_irsa_role_arn: "${ZARF_VAR_VELERO_S3_ROLE_ARN}"
+    velero_bucket: ${ZARF_VAR_VELERO_S3_BUCKET}
+    velero_bucket_region: ${ZARF_VAR_VELERO_S3_AWS_REGION}
+    velero_bucket_provider_url: ""
+    velero_bucket_credential_name: ""
+    velero_bucket_credential_key: ""

.github/test-infra/buckets-iac/loki.tf → .github/test-infra/aws/eks/loki.tf

@@ -25,4 +25,4 @@ resource "aws_iam_policy" "loki_policy" {
       }
     ]
   })
-}
+}

.github/test-infra/buckets-iac/main.tf → .github/test-infra/aws/eks/main.tf

@@ -1,29 +1,3 @@
-provider "aws" {
-  region = var.region
-
-  default_tags {
-    tags = {
-      PermissionsBoundary = var.permissions_boundary_name
-    }
-  }
-}
-
-terraform {
-  required_version = ">= 1.8.0"
-  backend "s3" {
-  }
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = ">= 4.0"
-    }
-
-    random = {
-      source  = "hashicorp/random"
-      version = "3.6.3"
-    }
-  }
-}
 
 resource "random_id" "default" {
   byte_length = 2
@@ -127,4 +101,4 @@ resource "aws_s3_bucket_policy" "bucket_policy" {
       }
     ]
   })
-}
+}

.github/test-infra/rds-iac/output.tf → .github/test-infra/aws/eks/outputs.tf

@@ -1,3 +1,31 @@
+output "aws_region" {
+  value = data.aws_region.current.name
+}
+
+output "loki_irsa_role_arn" {
+  value = module.irsa["loki"].role_arn
+}
+
+output "loki_s3" {
+  value = module.S3["loki"]
+}
+
+output "loki_s3_bucket" {
+  value = module.S3["loki"].bucket_name
+}
+
+output "velero_irsa_role_arn" {
+  value = module.irsa["velero"].role_arn
+}
+
+output "velero_s3" {
+  value = module.S3["velero"]
+}
+
+output "velero_s3_bucket" {
+  value = module.S3["velero"].bucket_name
+}
+
 output "grafana_pg_host" {
   description = "RDS Endpoint for Grafana"
   value       = element(split(":", module.db.db_instance_endpoint), 0)
@@ -23,3 +51,7 @@ output "grafana_pg_password" {
   value       = random_password.db_password.result
   sensitive   = true
 }
+
+output "grafana_ha" {
+  value = true
+}

.github/test-infra/rds-iac/main.tf → .github/test-infra/aws/eks/rds.tf

@@ -1,24 +1,3 @@
-provider "aws" {
-  region = var.region
-}
-
-terraform {
-  required_version = ">= 1.8.0"
-  backend "s3" {
-  }
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = ">= 4.0"
-    }
-
-    random = {
-      source  = "hashicorp/random"
-      version = "3.6.3"
-    }
-  }
-}
-
 resource "random_password" "db_password" {
   length  = 16
   special = false
@@ -105,14 +84,6 @@ data "aws_subnets" "subnets" {
   }
 }
 
-data "aws_partition" "current" {}
-
-data "aws_caller_identity" "current" {}
-
 locals {
   vpc_id = data.aws_vpc.vpc.id
 }
-
-resource "random_id" "unique_id" {
-  byte_length = 4
-}

.github/test-infra/aws/eks/variables.tf

@@ -0,0 +1,101 @@
+variable "region" {
+  description = "AWS region"
+  type        = string
+}
+
+variable "name" {
+  description = "Name for cluster"
+  type        = string
+}
+
+variable "permissions_boundary_name" {
+  description = "The name of the permissions boundary for IAM resources.  This will be used for tagging and to build out the ARN."
+  type        = string
+  default     = null
+}
+
+variable "use_permissions_boundary" {
+  description = "Whether to use IAM permissions boundary for resources."
+  type        = bool
+  default     = true
+}
+
+variable "key_owner_arns" {
+  description = "ARNS of KMS key owners, needed for use of key"
+  type        = list(string)
+  default     = []
+}
+
+# taken from zarf bb repo
+variable "kms_key_deletion_window" {
+  description = "Waiting period for scheduled KMS Key deletion. Can be 7-30 days."
+  type        = number
+  default     = 7
+}
+
+variable "bucket_configurations" {
+  type = map(object({
+    name            = string
+    service_account = string
+    namespace       = string
+  }))
+  default = {
+    loki = {
+      name            = "loki"
+      service_account = "loki"
+      namespace       = "loki"
+    }
+    velero = {
+      name            = "velero"
+      service_account = "velero-server"
+      namespace       = "velero"
+    }
+  }
+}
+
+variable "recovery_window" {
+  default = 7
+  type    = number
+}
+
+variable "db_name" {
+  description = "The name to give the database"
+  type        = string
+  default     = "grafana"
+}
+
+variable "db_port" {
+  description = "The database port"
+  type        = number
+  default     = 5432
+}
+
+variable "username" {
+  description = "The username to use to login to the DB"
+  type        = string
+  default     = "grafana"
+}
+
+variable "db_engine_version" {
+  description = "The Postgres engine version to use for the DB"
+  type        = string
+  default     = "15.7"
+}
+
+variable "db_allocated_storage" {
+  description = "Storage allocated to RDS instance"
+  type        = number
+  default     = 20
+}
+
+variable "db_storage_type" {
+  description = "The type of storage (e.g., gp2, io1)"
+  type        = string
+  default     = "gp2"
+}
+
+variable "db_instance_class" {
+  description = "The class of RDS instance (e.g., db.t4g.large)"
+  type        = string
+  default     = "db.t4g.large"
+}

.github/test-infra/buckets-iac/velero.tf → .github/test-infra/aws/eks/velero.tf

@@ -56,4 +56,3 @@ resource "aws_iam_policy" "velero_policy" {
       ]
   })
 }
-

.github/test-infra/aws/eks/versions.tf

@@ -0,0 +1,25 @@
+provider "aws" {
+  region = var.region
+
+  default_tags {
+    tags = {
+      PermissionsBoundary = var.permissions_boundary_name
+    }
+  }
+}
+terraform {
+  required_version = ">= 1.8.0"
+  backend "s3" {
+  }
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 4.0"
+    }
+
+    random = {
+      source  = "hashicorp/random"
+      version = "3.6.3"
+    }
+  }
+}