-
Notifications
You must be signed in to change notification settings - Fork 33
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introduce optional read-only mode examples (#185)
- Loading branch information
Showing
8 changed files
with
549 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
# Example of EKS cluster connected to CAST AI to read-only mode with ability to switch cluster to full access mode. | ||
Following example creates EKS cluster and its supporting resources using AWS community modules. | ||
After EKS cluster is created, by default it is onboarded to CAST AI in Read-only mode. | ||
Read-only behaviour is controlled by bool variable `readonly`. | ||
Example configuration should be analysed in the following order: | ||
1. Create VPC - `vpc.tf` | ||
2. Create EKS cluster - `eks.tf` | ||
3. Create IAM resources required for CAST AI full access mode to manage the cluster - `iam.tf` | ||
4. Create CAST AI related resources to connect EKS cluster to CAST AI in read-only mode - `castai.tf` | ||
|
||
# Usage | ||
1. Rename `tf.vars.example` to `tf.vars` | ||
2. Update `tf.vars` file with your cluster name, cluster region and CAST AI API token | ||
3. Initialize Terraform. Under example root folder run: | ||
``` | ||
terraform init | ||
``` | ||
4. Run Terraform apply: | ||
``` | ||
terraform apply -var-file=tf.vars | ||
``` | ||
5. To destroy resources created by this example: | ||
``` | ||
terraform destroy -var-file=tf.vars | ||
``` | ||
|
||
# Enable full access mode after cluster was already onboarded to read-only mode | ||
1. Set variable `readonly = false` and run `terraform apply -var-file=tf.vars` | ||
|
||
Note: Current provider version (v3.13.0) does not support `castai_eks_cluster` resource in-place re-creation. | ||
If `castai_eks_cluster` have to be re-created follow the steps below: | ||
1. Delete cluster using [/DeleteCluster](https://api.cast.ai/v1/spec/#/ExternalClusterAPI/ExternalClusterAPIDeleteCluster) API call. | ||
If cluster was already onboarded to full access mode, and you don't want to delete CAST AI created K8s nodes, make sure that `castai_eks_cluster` parameter is set to `delete_nodes_on_disconnect = false`. | ||
2. Run `terraform apply -var-file=tf.vars` | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,128 @@ | ||
data "aws_caller_identity" "current" {} | ||
|
||
provider "castai" { | ||
api_token = var.castai_api_token | ||
api_url = var.castai_api_url | ||
} | ||
|
||
provider "helm" { | ||
kubernetes { | ||
host = module.eks.cluster_endpoint | ||
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) | ||
exec { | ||
api_version = "client.authentication.k8s.io/v1beta1" | ||
command = "aws" | ||
# This requires the awscli to be installed locally where Terraform is executed. | ||
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", var.cluster_region] | ||
} | ||
} | ||
} | ||
|
||
resource "castai_eks_cluster" "this" { | ||
account_id = data.aws_caller_identity.current.account_id | ||
region = var.cluster_region | ||
name = var.cluster_name | ||
|
||
delete_nodes_on_disconnect = var.delete_nodes_on_disconnect | ||
assume_role_arn = var.readonly ? null: aws_iam_role.assume_role.arn | ||
} | ||
|
||
data "castai_eks_user_arn" "castai_user_arn" { | ||
cluster_id = castai_eks_clusterid.cluster_id.id | ||
} | ||
|
||
resource "castai_eks_clusterid" "cluster_id" { | ||
account_id = data.aws_caller_identity.current.account_id | ||
region = var.cluster_region | ||
cluster_name = var.cluster_name | ||
} | ||
|
||
|
||
resource "helm_release" "castai_agent" { | ||
name = "castai-agent" | ||
repository = "https://castai.github.io/helm-charts" | ||
chart = "castai-agent" | ||
namespace = "castai-agent" | ||
create_namespace = true | ||
cleanup_on_fail = true | ||
|
||
set { | ||
name = "provider" | ||
value = "eks" | ||
} | ||
set_sensitive { | ||
name = "apiKey" | ||
value = castai_eks_cluster.this.cluster_token | ||
} | ||
|
||
# Required until https://github.com/castai/helm-charts/issues/135 is fixed. | ||
set { | ||
name = "createNamespace" | ||
value = "false" | ||
} | ||
dynamic "set" { | ||
for_each = var.castai_api_url != "" ? [var.castai_api_url] : [] | ||
content { | ||
name = "apiURL" | ||
value = var.castai_api_url | ||
} | ||
} | ||
} | ||
|
||
resource "castai_node_configuration" "this" { | ||
count = var.readonly ? 0:1 | ||
cluster_id = castai_eks_cluster.this.id | ||
|
||
name = "default" | ||
disk_cpu_ratio = 0 | ||
min_disk_size = 100 | ||
subnets = module.vpc.private_subnets | ||
eks { | ||
security_groups = [ | ||
module.eks.cluster_security_group_id, | ||
module.eks.node_security_group_id, | ||
] | ||
instance_profile_arn = aws_iam_instance_profile.castai_instance_profile.arn | ||
} | ||
} | ||
|
||
resource "castai_node_configuration_default" "this" { | ||
count = var.readonly ? 0:1 | ||
cluster_id = castai_eks_cluster.this.id | ||
configuration_id = castai_node_configuration.this[0].id | ||
} | ||
|
||
resource "helm_release" "castai_cluster_controller" { | ||
count = var.readonly ? 0:1 | ||
name = "cluster-controller" | ||
repository = "https://castai.github.io/helm-charts" | ||
chart = "castai-cluster-controller" | ||
namespace = "castai-agent" | ||
create_namespace = true | ||
cleanup_on_fail = true | ||
wait = true | ||
|
||
set { | ||
name = "castai.clusterID" | ||
value = castai_eks_cluster.this.id | ||
} | ||
|
||
dynamic "set" { | ||
for_each = var.castai_api_url != "" ? [var.castai_api_url] : [] | ||
content { | ||
name = "castai.apiURL" | ||
value = var.castai_api_url | ||
} | ||
} | ||
|
||
set_sensitive { | ||
name = "castai.apiKey" | ||
value = castai_eks_cluster.this.cluster_token | ||
} | ||
|
||
depends_on = [helm_release.castai_agent] | ||
|
||
lifecycle { | ||
ignore_changes = [version] | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
# 2. Create EKS cluster. | ||
module "eks" { | ||
source = "terraform-aws-modules/eks/aws" | ||
version = "19.4.2" | ||
putin_khuylo = true | ||
|
||
cluster_name = var.cluster_name | ||
cluster_version = var.cluster_version | ||
cluster_endpoint_public_access = true | ||
|
||
cluster_addons = { | ||
coredns = { | ||
most_recent = true | ||
} | ||
kube-proxy = { | ||
most_recent = true | ||
} | ||
vpc-cni = { | ||
most_recent = true | ||
} | ||
} | ||
|
||
vpc_id = module.vpc.vpc_id | ||
subnet_ids = module.vpc.private_subnets | ||
|
||
# Self managed node groups will not automatically create the aws-auth configmap so we need to. | ||
create_aws_auth_configmap = true | ||
manage_aws_auth_configmap = true | ||
|
||
self_managed_node_groups = { | ||
node_group_1 = { | ||
name = "${var.cluster_name}-ng-1" | ||
instance_type = "m5.large" | ||
desired_size = 2 | ||
} | ||
} | ||
|
||
aws_auth_roles = var.readonly ? [] : [ | ||
# Add the CAST AI IAM role which required for CAST AI nodes to join the cluster. | ||
{ | ||
rolearn = aws_iam_role.castai_instance_profile_role.arn | ||
username = "system:node:{{EC2PrivateDNSName}}" | ||
groups = [ | ||
"system:bootstrappers", | ||
"system:nodes", | ||
] | ||
}, | ||
] | ||
|
||
} |
Oops, something went wrong.