diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index ef1a3eb..0000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# You can use any debian based image you want -ARG VARIANT="16-buster" -FROM mcr.microsoft.com/vscode/devcontainers/base:0-bullseye -# FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0-${VARIANT} - -ENV HECKA="foo boo" - -# Install openvpn client -RUN export DEBIAN_FRONTEND=noninteractive && apt-get update \ - && apt-get -y install --no-install-recommends openvpn \ - && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/library-scripts \ - # - # Remove the OPENVPN_CONFIG variable since we don't neeed it after is written to a file - && echo 'OPENVPN_CONFIG=""' >> /etc/environment \ - && echo "unset OPENVPN_CONFIG" | tee -a /etc/bash.bashrc > /etc/profile.d/999-unset-openvpn-config.sh \ - && if [ -d "/etc/zsh" ]; then echo "unset OPENVPN_CONFIG" >> /etc/zsh/zshenv; fi \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 1c996f9..0000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "MyCluster", - "build": { - "dockerfile": "Dockerfile" - }, - - // Allow the container to interact with host networking - "runArgs": ["--cap-add=NET_ADMIN", "--cap-add=NET_RAW", "--device=/dev/net/tun"], - - // Save the contents of the OPENVPN_CONFIG secret to disk - it lands in .devcontainer/openvpn-tmp - "initializeCommand": "bash .devcontainer/save-config.sh", - - // [Optional] Once the dev container is running, automatically start up the VPN client - "postStartCommand": "bash .devcontainer/start-openvpn.sh", - - // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" -} \ No newline at end of file diff --git a/.devcontainer/save-config.sh b/.devcontainer/save-config.sh deleted file mode 100644 index edad3b9..0000000 --- a/.devcontainer/save-config.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Switch to the .devcontainer folder -cd "$( dirname "${BASH_SOURCE[0]}" )" - -# Create a temporary directory -mkdir -p openvpn-tmp -cd openvpn-tmp - -# Save the configuration from the secret if it is present -if [ ! -z "${OPENVPN_CONFIG}" ]; then - echo "${OPENVPN_CONFIG}" > vpnconfig.ovpn -fi -if [ ! -z "${OPENVPN_AUTH}" ]; then - echo "${OPENVPN_AUTH}" > auth.txt -fi diff --git a/.devcontainer/start-openvpn.sh b/.devcontainer/start-openvpn.sh deleted file mode 100644 index c8bac90..0000000 --- a/.devcontainer/start-openvpn.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Switch to the .devcontainer folder -cd "$( dirname "${BASH_SOURCE[0]}" )" - -# Create a temporary directory -mkdir -p openvpn-tmp -cd openvpn-tmp - -# Touch file to make sure this user can read it -touch openvpn.log - -# If we are running as root, we do not need to use sudo -sudo_cmd="" -if [ "$(id -u)" != "0" ]; then - sudo_cmd="sudo" -fi - -# Start up the VPN client using the config stored in vpnconfig.ovpn by save-config.sh -nohup ${sudo_cmd} /bin/sh -c "openvpn --config vpnconfig.ovpn --log openvpn.log --auth-user-pass auth.txt &" | tee openvpn-launch.log diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml new file mode 100644 index 0000000..2c0e8b6 --- /dev/null +++ b/.github/workflows/infra.yml @@ -0,0 +1,86 @@ +name: SetUp the sales demo + +on: + workflow_dispatch: + inputs: + module: + description: Module to run + type: choice + default: foundation + options: + - admin-tenant + - aws-services + - app + cmd: + description: Command to run + type: choice + default: plan + options: + - plan + - apply + - destroy + environment: + description: Environment to deploy to + type: environment + default: salesdemo + workspace: + description: Workspace to run + type: string + default: default + + workflow_call: + inputs: + cmd: + description: Command to run + type: string + default: plan + environment: + description: Environment to deploy to + type: string + default: dev + module: + description: Module to run + type: string + default: admin-tenant + workspace: + description: Workspace to run + type: string + default: default + secrets: + DUPLO_TOKEN: + description: Duplo Token + required: true +jobs: + module: + name: ${{ inputs.cmd }} ${{ inputs.workspace }} foundation + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + env: + DUPLO_TOKEN: ${{ secrets.DUPLO_TOKEN }} + DUPLO_HOST: ${{ vars.DUPLO_HOST }} + DUPLO_TENANT: ${{ vars.DUPLO_TENANT }} + TF_CLI_ARGS_apply: -parallelism=1 + steps: + + - name: Checkout source code + uses: actions/checkout@v3 + + - name: Duplo and AWS Setup + uses: duplocloud/actions/setup@main + + - name: Terraform Setup + uses: duplocloud/actions/setup-terraform@main + + - name: TF Validate Module + uses: duplocloud/actions/terraform-module@main + with: + module: modules/foundation + test: false + + - name: TF Execute Module + uses: duplocloud/actions/terraform-exec@main + with: + module: iac/${{ inputs.module }} + workspace: ${{ inputs.workspace }} + command: ${{ inputs.cmd }} diff --git a/docs/Steps.md b/docs/Steps.md deleted file mode 100644 index c4af789..0000000 --- a/docs/Steps.md +++ /dev/null @@ -1,26 +0,0 @@ -# Steps to Build this Project - -## Step 01: Initialize Project - -Create a directory: - -```sh -mkdir duplo-pipeline -cd duplo-pipeline -``` - -Initialize a GIT repo and follow prompts: - -```sh -git init -``` - -Initialize npm project and follow prompts: - -```sh -npm init -``` - -## Step 02: Make an API - -Add code to [src](../src) directory. diff --git a/iac/.gitignore b/iac/.gitignore new file mode 100644 index 0000000..768fa60 --- /dev/null +++ b/iac/.gitignore @@ -0,0 +1,37 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log +duplo.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +# +#*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc +target/** \ No newline at end of file diff --git a/iac/.terraform-version b/iac/.terraform-version new file mode 100644 index 0000000..9075be4 --- /dev/null +++ b/iac/.terraform-version @@ -0,0 +1 @@ +1.5.5 diff --git a/iac/terraform/admin-tenant/.terraform.lock.hcl b/iac/terraform/admin-tenant/.terraform.lock.hcl new file mode 100644 index 0000000..dcdcaa0 --- /dev/null +++ b/iac/terraform/admin-tenant/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/duplocloud/duplocloud" { + version = "0.10.4" + constraints = "~> 0.10.2" + hashes = [ + "h1:ri0DBK0AMVOURqvykMnwh9thfHcKAcJtfz6i2tVhnRU=", + "zh:06596908bdbcce59bee2b7f430d4c744e936041bc3864e3cfcf33c58dcb01387", + "zh:1935730700e267a4c0eabb18105e7e3146c13609f5a75ba31269bac4118e4f7b", + "zh:35c2436281ec7d1580a2085901b030b71175e646bd2e51211c70636a150a117d", + "zh:56f2f7c34ace93fdd3209360ac2f3a7e1c20554e40dcbd594995759abbe54d63", + "zh:65533efff7720a0aa5f58b1c831aba230ea455fee8a50fd8cf33bafe05318cff", + "zh:912c4a863442dc1aa00d2d6f8027ee7f2f37792ca4da208a7dcd084040137424", + "zh:ae6d9d3f5fe80332f23cfe2dfc214cb3ee22e6d346a06722b476fed846708314", + "zh:b2a14f19b990317fbd3e64ff90af6e07d5f1c63a51340609bf4f963556da2627", + "zh:c9fa0106869bc9742b22e6afed0fe462a100d259013fa49b57414159f3acf0d4", + "zh:cce2a6ba1aff2157686ee7ba90f6c26acc09318f3047aefc830f50bb5a64e848", + "zh:e54c99c8e01f1ba2a1eda1364e2afbe15bce79c090c8f6f8f971775684ebb40d", + "zh:fdb533a8765072f6b499e9aedaaf84f45d4c0256861b960d0b3a3d651b1cad0a", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.32.1" + hashes = [ + "h1:jv1eeMPObgcJOEV6x9BmT1rSUDZ5cX0FBpweCNu4bnc=", + "zh:0c603e0ea9ec481f1588ca44d3464fe43ed936a8452e0c70d347c8e71a1b19a4", + "zh:0d43c845330ea4aaa152caf35819069215fcf17e4468b9d94c631f7d4178b1ac", + "zh:1211275208e8142bfa27987fdeb3eae40075ff569bf198330975f470bc4f5137", + "zh:1d8e7e4a2ff45a8b56037d030e2978fc04007941f62f1e265e251801a1d0c3cc", + "zh:4f6a8a6c9413b8b9267673cb7fb9dee7dc81946f7cc17d23e2104304f4ec4472", + "zh:6d769c74f8157260a37a32a1036b77f9795e21df2df7cadf4c7acc85b2dfd96e", + "zh:778fd9bf80424a62ebf5f059dcabfc4a588b0791ba18c1cf727bbdc1aed40351", + "zh:7bf1b063065bbe39b71e2a5895915fcbcc0cf7f553f84388e81888506d292fce", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b57506c3f46e850543fc1ee9522f231311e8540730db76bbf7a3f4d81777a4bd", + "zh:d37c8655b2a31435a116a1af7031f2bcdecf4c7e7e74903b88203798fb39043e", + "zh:db369802896eb10bbfed00bf3bd568b35fb5d903d3624d555b6574c5c4e2d94e", + "zh:e9992bfccf8205c495aebb7da917404496f96b5d3ea4a915a8884994ca8d860c", + "zh:ed1e0ef83cde313f1ccb3e18fc9dc63bf6ca473ec07554df5e24c706708a6866", + "zh:f0d19ed41352da9be308dff72899ecf5af7a42b592cf37fb98e9064e7622d35e", + ] +} diff --git a/iac/terraform/admin-tenant/backend.tf b/iac/terraform/admin-tenant/backend.tf new file mode 100644 index 0000000..2cc146f --- /dev/null +++ b/iac/terraform/admin-tenant/backend.tf @@ -0,0 +1,8 @@ +terraform { + backend "s3" { + region = "us-west-2" + key = "tenant" + workspace_key_prefix = "admin:" + encrypt = true + } +} diff --git a/iac/terraform/admin-tenant/main.tf b/iac/terraform/admin-tenant/main.tf new file mode 100644 index 0000000..68afbe2 --- /dev/null +++ b/iac/terraform/admin-tenant/main.tf @@ -0,0 +1,23 @@ +locals { + plan_id = var.infra_name + tenant_name = terraform.workspace + region = data.duplocloud_infrastructure.infra.region +} + +data "duplocloud_infrastructure" "infra" { + infra_name = var.infra_name +} + +resource "duplocloud_tenant" "tenant" { + account_name = local.tenant_name + plan_id = local.plan_id + allow_deletion = true +} + +resource "duplocloud_tenant_config" "tenant-config" { + tenant_id = duplocloud_tenant.tenant.tenant_id + setting { + key = "delete_protection" + value = "true" + } +} diff --git a/iac/terraform/admin-tenant/outputs.tf b/iac/terraform/admin-tenant/outputs.tf new file mode 100644 index 0000000..102ea3c --- /dev/null +++ b/iac/terraform/admin-tenant/outputs.tf @@ -0,0 +1,22 @@ +output "tenant_name" { + value = duplocloud_tenant.tenant.account_name + description = "The tenant name" +} +output "tenant_id" { + value = duplocloud_tenant.tenant.tenant_id + description = "The tenant ID" +} + +output "infra_name" { + value = data.duplocloud_infrastructure.infra.infra_name + description = "The duplo infra name." +} +output "vpc_id" { + value = data.duplocloud_infrastructure.infra.vpc_id + description = "The VPC or VNet ID." +} +output "region" { + value = local.region + description = "The AWS region the Tenant is located in" +} + diff --git a/iac/terraform/admin-tenant/providers.tf b/iac/terraform/admin-tenant/providers.tf new file mode 100644 index 0000000..c6ab923 --- /dev/null +++ b/iac/terraform/admin-tenant/providers.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.5.5" + required_providers { + duplocloud = { + source = "duplocloud/duplocloud" + version = "~> 0.10.2" + } + } +} +provider "duplocloud" { + +} +provider "aws" { + region = local.region +} diff --git a/iac/terraform/admin-tenant/vars.tf b/iac/terraform/admin-tenant/vars.tf new file mode 100644 index 0000000..bdac3fc --- /dev/null +++ b/iac/terraform/admin-tenant/vars.tf @@ -0,0 +1,8 @@ +variable "region" { + default = "us-west-2" + type = string +} +variable "infra_name" { + type = string + default = "prod-infra" +} diff --git a/iac/terraform/app/.terraform.lock.hcl b/iac/terraform/app/.terraform.lock.hcl new file mode 100644 index 0000000..86b869f --- /dev/null +++ b/iac/terraform/app/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/duplocloud/duplocloud" { + version = "0.10.5" + constraints = "~> 0.10.5" + hashes = [ + "h1:6COWVejHgI+ylqJDxJ5SqzOWyIaWL7CEOaUXBHBrwn4=", + "zh:009f8b97769e2e9fb2a35702811772896bd3378047e38035fc9e9183e3b652ea", + "zh:01f3bb97715d489583de6af5e34a4dd0fbbd00a04d1296cd2053c563d7aa94f1", + "zh:0ed23674f2965519fe32d917cb4069b67eb71fadb016fddddcee8475700ddae7", + "zh:3e1c1121a00228a1d748496eba8b580e5c5c2b3bf2a2562764692ae8aa5b21bd", + "zh:46c3976f1967e2fdda1529ba9cdf7d80886ffdd9107d434fbd6abd28606c3fd2", + "zh:50d6cda2521b48c08896dc438256644e06eea3e57b075f182c4415e56255b513", + "zh:5328d8f76bfda79318ca89e4f6a14d9977825801b4de9a0018977e869ec402bb", + "zh:5f67c8a356e3b7a7d674053f90f5ebc58c477562e2c98262a9c3a3024ece6a51", + "zh:96540c39084405696b822df6bf775e908ac83f6cb8699a1064be974e209f0b8c", + "zh:99f1c96281380cbd57332961783f58402708da1774f5cae6c2fbb4c6279024e7", + "zh:c24d77ffc06d30d5e3ab49ada7519f3bc219c00599fa24203a3e45846e1de530", + "zh:c7f1058a5ed8bdb62fe8f71d161abec28615b3a616737b68924ea3b0b984e7ba", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.34.0" + hashes = [ + "h1:1Y1JgV1z99QqAK06+atyfNqreZxyGZKbm4mZO4VhhT8=", + "zh:01bb20ae12b8c66f0cacec4f417a5d6741f018009f3a66077008e67cce127aa4", + "zh:3b0c9bdbbf846beef2c9573fc27898ceb71b69cf9d2f4b1dd2d0c2b539eab114", + "zh:5226ecb9c21c2f6fbf1d662ac82459ffcd4ad058a9ea9c6200750a21a80ca009", + "zh:6021b905d9b3cd3d7892eb04d405c6fa20112718de1d6ef7b9f1db0b0c97721a", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9e61b8e0ccf923979cd2dc1f1140dbcb02f92248578e10c1996f560b6306317c", + "zh:ad6bf62cdcf531f2f92f6416822918b7ba2af298e4a0065c6baf44991fda982d", + "zh:b698b041ef38837753bbe5265dddbc70b76e8b8b34c5c10876e6aab0eb5eaf63", + "zh:bb799843c534f6a3f072a99d93a3b53ff97c58a96742be15518adf8127706784", + "zh:cebee0d942c37cd3b21e9050457cceb26d0a6ea886b855dab64bb67d78f863d1", + "zh:e061fdd1cb99e7c81fb4485b41ae000c6792d38f73f9f50aed0d3d5c2ce6dcfb", + "zh:eeb4943f82734946362696928336357cd1d36164907ae5905da0316a67e275e1", + "zh:ef09b6ad475efa9300327a30cbbe4373d817261c8e41e5b7391750b16ef4547d", + "zh:f01aab3881cd90b3f56da7c2a75f83da37fd03cc615fc5600a44056a7e0f9af7", + "zh:fcd0f724ebc4b56a499eb6c0fc602de609af18a0d578befa2f7a8df155c55550", + ] +} diff --git a/iac/terraform/app/alb.tf b/iac/terraform/app/alb.tf new file mode 100644 index 0000000..af4dcfe --- /dev/null +++ b/iac/terraform/app/alb.tf @@ -0,0 +1,208 @@ + +resource "duplocloud_aws_load_balancer" "askduplo_dev" { + tenant_id = local.tenant_id + name = "search" + load_balancer_type = "application" + enable_access_logs = false + is_internal = false + drop_invalid_headers = false + http_to_https_redirect = true + idle_timeout = 60 +} + +resource "duplocloud_aws_load_balancer_listener" "askduplo_dev_listener_443" { + tenant_id = local.tenant_id + load_balancer_name = duplocloud_aws_load_balancer.askduplo_dev.name + certificate_arn = local.cert_arn + protocol = "HTTPS" + port = 443 + target_group_arn = duplocloud_duplo_service_lbconfigs.frontend_config.lbconfigs[0].target_group_arn +} + +resource "duplocloud_aws_target_group_attributes" "askduplo_dev_listener_443_tg_attributes" { + tenant_id = local.tenant_id + target_group_arn = duplocloud_aws_load_balancer_listener.askduplo_dev_listener_443.target_group_arn + attribute { + key = "target_group_health.unhealthy_state_routing.minimum_healthy_targets.count" + value = "1" + } + attribute { + key = "stickiness.enabled" + value = "false" + } + attribute { + key = "load_balancing.algorithm.anomaly_mitigation" + value = "off" + } + attribute { + key = "target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage" + value = "off" + } + attribute { + key = "deregistration_delay.timeout_seconds" + value = "300" + } + attribute { + key = "target_group_health.dns_failover.minimum_healthy_targets.count" + value = "1" + } + attribute { + key = "stickiness.type" + value = "lb_cookie" + } + attribute { + key = "stickiness.lb_cookie.duration_seconds" + value = "86400" + } + attribute { + key = "slow_start.duration_seconds" + value = "0" + } + attribute { + key = "stickiness.app_cookie.duration_seconds" + value = "86400" + } + attribute { + key = "target_group_health.dns_failover.minimum_healthy_targets.percentage" + value = "off" + } + attribute { + key = "load_balancing.cross_zone.enabled" + value = "use_load_balancer_configuration" + } + attribute { + key = "load_balancing.algorithm.type" + value = "round_robin" + } +} +resource "duplocloud_aws_load_balancer_listener" "askduplo_dev_listener_5000" { + tenant_id = local.tenant_id + load_balancer_name = duplocloud_aws_load_balancer.askduplo_dev.name + certificate_arn = local.cert_arn + protocol = "HTTPS" + port = 5000 + target_group_arn = duplocloud_duplo_service_lbconfigs.backend_config.lbconfigs[0].target_group_arn +} + +resource "duplocloud_aws_target_group_attributes" "askduplo_dev_listener_5000_tg_attributes" { + tenant_id = local.tenant_id + target_group_arn = duplocloud_aws_load_balancer_listener.askduplo_dev_listener_5000.target_group_arn + attribute { + key = "target_group_health.unhealthy_state_routing.minimum_healthy_targets.count" + value = "1" + } + attribute { + key = "stickiness.enabled" + value = "false" + } + attribute { + key = "load_balancing.algorithm.anomaly_mitigation" + value = "off" + } + attribute { + key = "target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage" + value = "off" + } + attribute { + key = "deregistration_delay.timeout_seconds" + value = "300" + } + attribute { + key = "target_group_health.dns_failover.minimum_healthy_targets.count" + value = "1" + } + attribute { + key = "stickiness.type" + value = "lb_cookie" + } + attribute { + key = "stickiness.lb_cookie.duration_seconds" + value = "86400" + } + attribute { + key = "slow_start.duration_seconds" + value = "0" + } + attribute { + key = "stickiness.app_cookie.duration_seconds" + value = "86400" + } + attribute { + key = "target_group_health.dns_failover.minimum_healthy_targets.percentage" + value = "off" + } + attribute { + key = "load_balancing.cross_zone.enabled" + value = "use_load_balancer_configuration" + } + attribute { + key = "load_balancing.algorithm.type" + value = "round_robin" + } +} + +resource "duplocloud_aws_load_balancer_listener" "slack_listener_3000" { + tenant_id = local.tenant_id + load_balancer_name = duplocloud_aws_load_balancer.askduplo_dev.name + certificate_arn = local.cert_arn + protocol = "HTTPS" + port = 3000 + target_group_arn = duplocloud_duplo_service_lbconfigs.slack_config.lbconfigs[0].target_group_arn +} + +resource "duplocloud_aws_target_group_attributes" "slack_listener_3000_tg_attributes" { + tenant_id = local.tenant_id + target_group_arn = duplocloud_aws_load_balancer_listener.slack_listener_3000.target_group_arn + attribute { + key = "target_group_health.unhealthy_state_routing.minimum_healthy_targets.count" + value = "1" + } + attribute { + key = "stickiness.enabled" + value = "false" + } + attribute { + key = "load_balancing.algorithm.anomaly_mitigation" + value = "off" + } + attribute { + key = "target_group_health.unhealthy_state_routing.minimum_healthy_targets.percentage" + value = "off" + } + attribute { + key = "deregistration_delay.timeout_seconds" + value = "300" + } + attribute { + key = "target_group_health.dns_failover.minimum_healthy_targets.count" + value = "1" + } + attribute { + key = "stickiness.type" + value = "lb_cookie" + } + attribute { + key = "stickiness.lb_cookie.duration_seconds" + value = "86400" + } + attribute { + key = "slow_start.duration_seconds" + value = "0" + } + attribute { + key = "stickiness.app_cookie.duration_seconds" + value = "86400" + } + attribute { + key = "target_group_health.dns_failover.minimum_healthy_targets.percentage" + value = "off" + } + attribute { + key = "load_balancing.cross_zone.enabled" + value = "use_load_balancer_configuration" + } + attribute { + key = "load_balancing.algorithm.type" + value = "round_robin" + } +} diff --git a/iac/terraform/app/backend.tf b/iac/terraform/app/backend.tf new file mode 100644 index 0000000..db3cdcd --- /dev/null +++ b/iac/terraform/app/backend.tf @@ -0,0 +1,8 @@ +terraform { + backend "s3" { + region = "us-west-2" + key = "app" + workspace_key_prefix = "tenant:" + encrypt = true + } +} diff --git a/iac/terraform/app/k8s-secret-slack.tf b/iac/terraform/app/k8s-secret-slack.tf new file mode 100644 index 0000000..eb6af39 --- /dev/null +++ b/iac/terraform/app/k8s-secret-slack.tf @@ -0,0 +1,43 @@ + +# we have two slack bots, prod and non prod. In Duplo, all non prod environments (dev, demo, marketting, etc), +# will use the nonprod slack bot +resource "duplocloud_k8_secret" "slack" { + tenant_id = local.tenant_id + secret_name = "slack" + secret_type = "Opaque" + secret_data = jsonencode({ + "SLACK_BOT_TOKEN" : var.tenant == "ask-prod" ? var.prod_slack_bot_token : var.nonprod_slack_bot_token + "SLACK_SIGNING_SECRET" : var.tenant == "ask-prod" ? var.prod_slack_signing_secret : var.nonprod_slack_signing_secret + } + ) +} + +resource "duplocloud_k8_secret" "openai" { + tenant_id = local.tenant_id + secret_name = "openai" + secret_type = "Opaque" + secret_data = jsonencode({ + "OPENAI_API_KEY" : var.openai_api_key + } + ) +} + +resource "duplocloud_k8_secret" "clickup" { + tenant_id = local.tenant_id + secret_name = "clickup" + secret_type = "Opaque" + secret_data = jsonencode({ + "CLICKUP_TOKEN" : var.clickup_api_key + } + ) +} + +resource "duplocloud_k8_secret" "pinecone" { + tenant_id = local.tenant_id + secret_name = "pinecone" + secret_type = "Opaque" + secret_data = jsonencode({ + "PINECONE_API_KEY" : var.tenant == "ask-prod" ? var.prod_pinecone_api_key : var.nonprod_pinecone_api_key + } + ) +} diff --git a/iac/terraform/app/main.tf b/iac/terraform/app/main.tf new file mode 100644 index 0000000..273a67c --- /dev/null +++ b/iac/terraform/app/main.tf @@ -0,0 +1,54 @@ +data "aws_caller_identity" "current" { +} + +data "duplocloud_tenant_aws_region" "current" { + tenant_id = local.tenant_id +} + +locals { + tfstate_bucket = "duplo-tfstate-${data.aws_caller_identity.current.account_id}" + region = data.duplocloud_tenant_aws_region.current.aws_region + tenant_id = data.terraform_remote_state.tenant.outputs["tenant_id"] + cert_arn = data.terraform_remote_state.tenant.outputs["cert_arn"] + tenant_name = data.terraform_remote_state.tenant.outputs["tenant_name"] + sqs_queue_url = data.terraform_remote_state.aws-services.outputs["sqs_queue_url"] + sqs_queue_fullname = data.terraform_remote_state.aws-services.outputs["sqs_queue_fullname"] + sqs_queue_arn = data.terraform_remote_state.aws-services.outputs["sqs_queue_arn"] +} + +data "terraform_remote_state" "tenant" { + backend = "s3" + workspace = terraform.workspace + config = { + bucket = local.tfstate_bucket + workspace_key_prefix = "admin:" + key = "tenant" + region = "us-west-2" + } +} + +data "terraform_remote_state" "aws-services" { + backend = "s3" + workspace = terraform.workspace + config = { + bucket = local.tfstate_bucket + workspace_key_prefix = "tenant:" + key = "aws-services" + region = "us-west-2" + } +} + +data "duplocloud_tenant" "tenant" { + name = var.tenant +} + +data "duplocloud_tenant_aws_credentials" "test" { tenant_id = data.duplocloud_tenant.tenant.id } + +data "duplocloud_admin_aws_credentials" "this" {} + +provider "aws" { + access_key = data.duplocloud_admin_aws_credentials.this.access_key_id + secret_key = data.duplocloud_admin_aws_credentials.this.secret_access_key + token = data.duplocloud_admin_aws_credentials.this.session_token + region = "us-west-2" +} diff --git a/iac/terraform/app/outputs.tf b/iac/terraform/app/outputs.tf new file mode 100644 index 0000000..8ae3efd --- /dev/null +++ b/iac/terraform/app/outputs.tf @@ -0,0 +1,6 @@ + +output "url" { + value = "${var.tenant}.${var.base_domain}" + description = "URL for accessing the frontend" +} + diff --git a/iac/terraform/app/providers.tf b/iac/terraform/app/providers.tf new file mode 100644 index 0000000..cac0941 --- /dev/null +++ b/iac/terraform/app/providers.tf @@ -0,0 +1,12 @@ +terraform { + required_version = ">= 1.5.5" + required_providers { + duplocloud = { + source = "duplocloud/duplocloud" + version = "~> 0.10.5" + } + } +} +provider "duplocloud" { + +} diff --git a/iac/terraform/app/route53.tf b/iac/terraform/app/route53.tf new file mode 100644 index 0000000..0a7fa14 --- /dev/null +++ b/iac/terraform/app/route53.tf @@ -0,0 +1,42 @@ + +data "aws_route53_zone" "route53" { + name = var.base_domain + private_zone = false +} + +data "aws_lb" "lb" { + name = duplocloud_aws_load_balancer.askduplo_dev.fullname +} + +resource "aws_route53_record" "alias" { + zone_id = data.aws_route53_zone.route53.zone_id + name = var.tenant + type = "A" + + alias { + name = duplocloud_aws_load_balancer.askduplo_dev.dns_name + zone_id = data.aws_lb.lb.zone_id + evaluate_target_health = false + } +} + +# production, that is running on a VM, uses the following URL for the backend +# https://search.prod-apps.duplocloud.net:5000 +# To test the container production environment, and to enable quick rollback +# I am going to start controlling the value for this DNS record in this TF + +resource "aws_route53_record" "alias-old" { + + # only create this resource if we are dealing with the production environment + count = var.tenant == "ask-prod" ? 1 : 0 + + zone_id = data.aws_route53_zone.route53.zone_id + name = "search" + type = "A" + + alias { + name = var.prod_containerized == false ? "duplo3-search-search-1016995356.us-west-2.elb.amazonaws.com" : duplocloud_aws_load_balancer.askduplo_dev.dns_name + zone_id = data.aws_lb.lb.zone_id + evaluate_target_health = false + } +} \ No newline at end of file diff --git a/iac/terraform/app/scheduler.tf b/iac/terraform/app/scheduler.tf new file mode 100644 index 0000000..2c0b261 --- /dev/null +++ b/iac/terraform/app/scheduler.tf @@ -0,0 +1,123 @@ + +resource "duplocloud_k8_secret" "duplo_token" { + count = var.enable_scheduler ? 1 : 0 + tenant_id = local.tenant_id + secret_name = "duplo-token" + secret_type = "Opaque" + secret_data = jsonencode({ + "DUPLO_TOKEN" : var.duplo_token + } + ) +} + +resource "duplocloud_k8s_cron_job" "stop" { + count = var.enable_scheduler ? 1 : 0 + tenant_id = local.tenant_id + metadata { + name = "stop" + } + spec { + job_template { + spec { + ttl_seconds_after_finished = "172800" + template { + spec { + restart_policy = "Never" + container { + name = "scheduler" + image = "227120241369.dkr.ecr.us-west-2.amazonaws.com/scheduler:latest" + image_pull_policy = "Always" + command = ["python", "./main.py"] + env { + name = "duplo_action" + value = "stop" + } + + env { + name = "duplo_host" + value = "https://prod.duplocloud.net" + } + env { + name = "tenant_names" + value = var.scheduler_tenants + } + env { + name = "duplo_token" + value_from { + secret_key_ref { + key = "DUPLO_TOKEN" + name = "duplo-token" + } + } + } + } + } + } + } + } + schedule = var.shutdown_schedule + concurrency_policy = "Forbid" + failed_jobs_history_limit = 10 + successful_jobs_history_limit = 10 + + } +} + +resource "duplocloud_k8s_cron_job" "start" { + count = var.enable_scheduler ? 1 : 0 + tenant_id = local.tenant_id + metadata { + name = "start" + } + spec { + job_template { + spec { + ttl_seconds_after_finished = "172800" + template { + spec { + restart_policy = "Never" + container { + name = "scheduler" + image = "227120241369.dkr.ecr.us-west-2.amazonaws.com/scheduler:latest" + image_pull_policy = "Always" + command = ["python", "./main.py"] + env { + name = "duplo_action" + value = "start" + } + + env { + name = "duplo_host" + value = "https://prod.duplocloud.net" + } + env { + name = "tenant_names" + value = var.scheduler_tenants + } + env { + name = "duplo_token" + value_from { + secret_key_ref { + key = "DUPLO_TOKEN" + name = "duplo-token" + } + } + } + } + } + } + } + } + schedule = var.startup_schedule + concurrency_policy = "Forbid" + failed_jobs_history_limit = 10 + successful_jobs_history_limit = 10 + + } + + lifecycle { + ignore_changes = [ + spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image + ] + } +} diff --git a/iac/terraform/app/svc-backend.tf b/iac/terraform/app/svc-backend.tf new file mode 100644 index 0000000..1e39ea7 --- /dev/null +++ b/iac/terraform/app/svc-backend.tf @@ -0,0 +1,167 @@ +resource "duplocloud_duplo_service" "backend" { + tenant_id = local.tenant_id + name = "backend" + replicas = var.replica_count + lb_synced_deployment = false + cloud_creds_from_k8s_service_account = false + is_daemonset = false + is_unique_k8s_node_required = true + agent_platform = 7 + cloud = 0 + other_docker_config = jsonencode({ + "Env" : [ + { + "name" : "OPENAI_API_KEY", + "valueFrom" : { + "secretKeyRef" : { + "key" : "OPENAI_API_KEY", + "name" : "openai" + } + } + }, + { + "Name" : "PERSIST_DIRECTORY", + "Value" : "db" + }, + { + "Name" : "MODEL_TYPE", + "Value" : "GPT4All" + }, + { + "Name" : "MODEL_PATH", + "Value" : "models/ggml-gpt4all-j-v1.3-groovy.bin" + }, + { + "Name" : "EMBEDDINGS_MODEL_NAME", + "Value" : "all-MiniLM-L6-v2" + }, + { + "Name" : "MODEL_N_CTX", + "Value" : 1000 + }, + { + "Name" : "DUPLO", + "Value" : true + }, + { + "Name" : "S3_BUCKET", + "Value" : data.terraform_remote_state.aws-services.outputs["s3_search_fullname"] + }, + { + "Name" : "DUPLO_FINE_TUNED_MODEL", + "Value" : var.duplo_fine_tuned_model + }, + { + "Name" : "DUPLO_NATIVE_CONTEXT", + "Value" : var.duplo_native_context + }, + { + "name" : "PINECONE_API_KEY", + "valueFrom" : { + "secretKeyRef" : { + "key" : "PINECONE_API_KEY", + "name" : "pinecone" + } + } + }, + { + "Name" : "PINECONE_ENV", + "Value" : terraform.workspace == "ask-prod" ? "us-central1-gcp" : "gcp-starter" + }, + { + "Name" : "PINECONE_INDEX_NAME", + "Value" : terraform.workspace == "ask-prod" ? "askduplocloud-prod" : "askduplocloud-dev" + }, + { + "Name" : "SQS_QUEUE_URL", + "Value" : local.sqs_queue_url + }, + { + "Name" : "AWS_REGION", + "Value" : local.region + }, + { + "Name" : "SQS_QUEUE_FULLNAME", + "Value" : local.sqs_queue_fullname + }, + { + "Name" : "SQS_QUEUE_ARN", + "Value" : local.sqs_queue_arn + }, + + ], + "DeploymentStrategy" : { + "RollingUpdate" : { + "MaxSurge" : 1, + "MaxUnavailable" : 0 + } + }, + "startupProbe" : { + "httpGet" : { + "path" : "/health", + "port" : 5000 + }, + "initialDelaySeconds" : 60, + "periodSeconds" : 10, + "failureThreshold" : 25 + }, + "ReadinessProbe" : { + "httpGet" : { + "path" : "/health", + "port" : 5000 + }, + "periodSeconds" : 5, + "failureThreshold": 2 + }, + "LivenessProbe" : { + "httpGet" : { + "path" : "/health", + "port" : 5000 + }, + "periodSeconds" : 5, + "failureThreshold": 2 + }, + + # "PodSecurityContext" : { + # "FsGroup" : 1000, + # "RunAsGroup" : 1000, + # "RunAsUser" : 1000, + # "allowPrivilegeEscalation": false + # } + } + ) + + docker_image = var.svc_backend_docker_image + volumes = jsonencode([ + { + "Name" : "storage", + "Path" : "/app/storage", + "Spec" : { + "PersistentVolumeClaim" : { + "claimName" : "storage" + } + } + } + ] + ) + + lifecycle { + ignore_changes = [ + docker_image + ] + } +} + +resource "duplocloud_duplo_service_lbconfigs" "backend_config" { + tenant_id = duplocloud_duplo_service.backend.tenant_id + replication_controller_name = duplocloud_duplo_service.backend.name + lbconfigs { + lb_type = 7 + is_native = false + is_internal = false + port = 5000 + external_port = 5000 + protocol = "http" + health_check_url = "/health" + } +} \ No newline at end of file diff --git a/iac/terraform/app/svc-frontend.tf b/iac/terraform/app/svc-frontend.tf new file mode 100644 index 0000000..66e0860 --- /dev/null +++ b/iac/terraform/app/svc-frontend.tf @@ -0,0 +1,40 @@ +resource "duplocloud_duplo_service" "frontend" { + tenant_id = local.tenant_id + name = "frontend" + replicas = var.replica_count + lb_synced_deployment = false + cloud_creds_from_k8s_service_account = false + is_daemonset = false + is_unique_k8s_node_required = true + agent_platform = 7 + cloud = 0 + other_docker_config = jsonencode({ + "Env" : [ + { + "Name" : "DUPLO", + "Value" : "true" + } + ] + } + ) + docker_image = var.svc_frontend_docker_image + lifecycle { + ignore_changes = [ + docker_image + ] + } +} + +resource "duplocloud_duplo_service_lbconfigs" "frontend_config" { + tenant_id = duplocloud_duplo_service.frontend.tenant_id + replication_controller_name = duplocloud_duplo_service.frontend.name + lbconfigs { + lb_type = 7 + is_native = false + is_internal = false + port = 3000 + external_port = 3000 + protocol = "http" + health_check_url = "/" + } +} diff --git a/iac/terraform/app/svc-queue.tf b/iac/terraform/app/svc-queue.tf new file mode 100644 index 0000000..8860e20 --- /dev/null +++ b/iac/terraform/app/svc-queue.tf @@ -0,0 +1,100 @@ + +# https://www.middlewareinventory.com/blog/kubernetes-cronjob-best-practices/ + +resource "duplocloud_k8s_cron_job" "queue" { + tenant_id = local.tenant_id + metadata { + name = "queue-processor" + } + spec { + job_template { + spec { + parallelism = var.parallelism + completions = var.completions + backoff_limit = var.backoff_limit + active_deadline_seconds = var.active_deadline_seconds + ttl_seconds_after_finished = var.ttl_seconds_after_finished + template { + spec { + restart_policy = "Never" + container { + name = "scheduler" + image = var.svc_queue_docker_image + image_pull_policy = "Always" + + env { + name = "CONTAINER_ROLE" + value = "queue" + } + env { + name = "DUPLO" + value = "true" + } + + env { + name = "SQS_QUEUE_URL" + value = local.sqs_queue_url + } + env { + name = "AWS_REGION" + value = local.region + } + env { + name = "SQS_QUEUE_FULLNAME" + value = local.sqs_queue_fullname + } + env { + name = "SQS_QUEUE_ARN" + value = local.sqs_queue_arn + } + + env { + name = "S3_BUCKET" + value = data.terraform_remote_state.aws-services.outputs["s3_search_fullname"] + } + env { + name = "PINECONE_API_KEY" + value_from { + secret_key_ref { + key = "PINECONE_API_KEY" + name = "pinecone" + } + } + } + env { + name = "OPENAI_API_KEY" + value_from { + secret_key_ref { + key = "OPENAI_API_KEY" + name = "openai" + } + } + } + env { + name = "PINECONE_ENV" + value = terraform.workspace == "ask-prod" ? "us-central1-gcp" : "gcp-starter" + } + env { + name = "PINECONE_INDEX_NAME" + value = terraform.workspace == "ask-prod" ? "askduplocloud-prod" : "askduplocloud-dev" + } + + + + } + } + } + } + } + schedule = var.queue_schedule + concurrency_policy = var.concurrency_policy + failed_jobs_history_limit = var.failed_jobs_history_limit + successful_jobs_history_limit = var.successful_jobs_history_limit + starting_deadline_seconds = var.starting_deadline_seconds + } + lifecycle { + ignore_changes = [ + spec[0].job_template[0].spec[0].template[0].spec[0].container[0].image + ] + } +} diff --git a/iac/terraform/app/svc-slack.tf b/iac/terraform/app/svc-slack.tf new file mode 100644 index 0000000..5409a45 --- /dev/null +++ b/iac/terraform/app/svc-slack.tf @@ -0,0 +1,98 @@ +resource "duplocloud_duplo_service" "slack" { + tenant_id = local.tenant_id + name = "slack" + replicas = var.replica_count + lb_synced_deployment = false + cloud_creds_from_k8s_service_account = false + is_daemonset = false + is_unique_k8s_node_required = true + agent_platform = 7 + cloud = 0 + other_docker_config = jsonencode({ + "Env" : [ + { + "Name" : "KB_SAVE_BUCKET", + "Value" : data.terraform_remote_state.aws-services.outputs["s3_search_fullname"] + }, + { + "Name" : "DUPLO", + "Value" : true + }, + { + "Name" : "ASK_DUPLO_URL", + "Value" : "http://backend:5000/get_answer" + }, + { + "name" : "OPENAI_API_KEY", + "valueFrom" : { + "secretKeyRef" : { + "key" : "OPENAI_API_KEY", + "name" : "openai" + } + } + }, + { + "name" : "SLACK_BOT_TOKEN", + "valueFrom" : { + "secretKeyRef" : { + "key" : "SLACK_BOT_TOKEN", + "name" : "slack" + } + } + }, + { + "name" : "SLACK_SIGNING_SECRET", + "valueFrom" : { + "secretKeyRef" : { + "key" : "SLACK_SIGNING_SECRET", + "name" : "slack" + } + } + }, + { + "name" : "CLICKUP_TOKEN", + "valueFrom" : { + "secretKeyRef" : { + "key" : "CLICKUP_TOKEN", + "name" : "clickup" + } + } + }, + { + "Name" : "CLICKUP_LIST_ID", + "Value" : terraform.workspace == "ask-prod" ? var.prod_clickup_list_id : var.nonprod_clickup_list_id + }, + { + "Name" : "SLACK_ALLOWED_MEMBER_IDS", + "Value" : "${join(", ", var.slackbot_allowed_member_ids)}" + }, + { + "Name" : "SLACK_ALLOWED_CHANNEL_IDS", # in ask-dev pass in allowed channel IDs, ask-prod is set to "" which allows it to be used in all channels + "Value" : "${join(", ", terraform.workspace == "ask-dev" ? var.slackbot_allowed_channel_ids : [""])}" + } + + ] + } + ) + docker_image = var.svc_slack_docker_image + + lifecycle { + ignore_changes = [ + docker_image + ] + } +} + +resource "duplocloud_duplo_service_lbconfigs" "slack_config" { + tenant_id = duplocloud_duplo_service.slack.tenant_id + replication_controller_name = duplocloud_duplo_service.slack.name + lbconfigs { + lb_type = 7 + is_native = false + is_internal = false + port = 3000 + external_port = 3000 + protocol = "http" + health_check_url = "/health" + } +} diff --git a/iac/terraform/app/vars.tf b/iac/terraform/app/vars.tf new file mode 100644 index 0000000..9357bee --- /dev/null +++ b/iac/terraform/app/vars.tf @@ -0,0 +1,207 @@ + +variable "svc_frontend_docker_image" { + default = "227120241369.dkr.ecr.us-west-2.amazonaws.com/frontend:297ea03" + type = string +} + +variable "svc_backend_docker_image" { + default = "227120241369.dkr.ecr.us-west-2.amazonaws.com/backend:73f536c" + type = string +} + +variable "svc_slack_docker_image" { + default = "227120241369.dkr.ecr.us-west-2.amazonaws.com/slack:e93aeb8" + type = string +} + +variable "svc_queue_docker_image" { + default = "227120241369.dkr.ecr.us-west-2.amazonaws.com/backend:26eb99c" + type = string +} + +variable "replica_count" { + type = number + default = 1 +} + +# TODO: this is kind of a hack. I updated the apply.sh wrapper script to generate this value using TF_VAR_tenant +variable "tenant" { + type = string +} + +variable "base_domain" { + default = "prod-apps.duplocloud.net." + type = string +} + +variable "openai_api_key" { + default = "" + type = string + sensitive = true +} + +variable "duplo_token" { + default = "" + type = string + sensitive = true +} + +variable "duplo_fine_tuned_model" { + default = "ft:gpt-3.5-turbo-0613:duplocloud:mvp-v3-5-0613:8IVbPUuu" + type = string +} + +variable "duplo_native_context" { + default = "Answer the question as truthfully as possible, and if you're unsure of the answer, say 'Sorry, I don't know the answer to that question.'" + type = string +} + +variable "nonprod_slack_bot_token" { + default = "" + type = string + sensitive = true +} + +variable "nonprod_slack_signing_secret" { + default = "" + type = string + sensitive = true +} + +variable "prod_slack_bot_token" { + default = "" + type = string + sensitive = true +} + +variable "prod_slack_signing_secret" { + default = "" + type = string + sensitive = true +} + +variable "prod_containerized" { + default = false + type = bool +} + +variable "clickup_api_key" { + default = "" + type = string + sensitive = true +} + +variable "prod_clickup_list_id" { + default = "901301536541" + type = string + sensitive = true +} + +variable "nonprod_clickup_list_id" { + default = "901301484125" + type = string +} + +variable "slackbot_allowed_member_ids" { + default = [ + "U05P4ALN38D", # Pranav Chakilam + "U04MAA24JU8", # Andy Boutte + "U0353K34QKT", # Zafar Abbas + "U05HFPMLD4Y", # Brian Rawlins + "U03ALJH7HNC", # Raghavan + "U063T1RE8KX", # Cory May + "U06423XQMSM" # Luke Baker + ] + type = list +} + +variable "slackbot_allowed_channel_ids" { + default = [ + "C06AG8BPM0E" # ask-dev slack channel + ] + type = list +} + +variable "nonprod_pinecone_api_key" { + default = "" + type = string + sensitive = true +} + +variable "prod_pinecone_api_key" { + default = "" + type = string + sensitive = true +} + +variable "enable_scheduler" { + type = bool + default = false # default to false because we will only run this in prod +} + +variable "scheduler_tenants" { + description = "List of tenants the scheduler should run against" + type = string + default = "" +} + +variable "shutdown_schedule" { + default = "0 1 * * * " # 7pm central time = 1am UTC + type = string +} + +variable "startup_schedule" { + default = "0 13 * * *" # 7am central time = 1pm UTC + type = string +} + +variable "queue_schedule" { + default = "*/5 * * * *" + type = string +} + +variable "ttl_seconds_after_finished" { + default = "3600" + type = string +} + +variable "concurrency_policy" { + default = "Allow" + type = string +} + +variable "failed_jobs_history_limit" { + default = "10" + type = string +} + +variable "successful_jobs_history_limit" { + default = "10" + type = string +} + +variable "parallelism" { + default = "2" + type = string +} + +variable "starting_deadline_seconds" { + default = "60" + type = string +} + +variable "completions" { + default = "1" + type = string +} + +variable "backoff_limit" { + default = "2" + type = string +} + +variable "active_deadline_seconds" { + default = "300" + type = string +} + diff --git a/iac/terraform/aws-services/.terraform.lock.hcl b/iac/terraform/aws-services/.terraform.lock.hcl new file mode 100644 index 0000000..e2aaf5a --- /dev/null +++ b/iac/terraform/aws-services/.terraform.lock.hcl @@ -0,0 +1,65 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/duplocloud/duplocloud" { + version = "0.10.4" + constraints = "> 0.9.40, ~> 0.10.2" + hashes = [ + "h1:ri0DBK0AMVOURqvykMnwh9thfHcKAcJtfz6i2tVhnRU=", + "zh:06596908bdbcce59bee2b7f430d4c744e936041bc3864e3cfcf33c58dcb01387", + "zh:1935730700e267a4c0eabb18105e7e3146c13609f5a75ba31269bac4118e4f7b", + "zh:35c2436281ec7d1580a2085901b030b71175e646bd2e51211c70636a150a117d", + "zh:56f2f7c34ace93fdd3209360ac2f3a7e1c20554e40dcbd594995759abbe54d63", + "zh:65533efff7720a0aa5f58b1c831aba230ea455fee8a50fd8cf33bafe05318cff", + "zh:912c4a863442dc1aa00d2d6f8027ee7f2f37792ca4da208a7dcd084040137424", + "zh:ae6d9d3f5fe80332f23cfe2dfc214cb3ee22e6d346a06722b476fed846708314", + "zh:b2a14f19b990317fbd3e64ff90af6e07d5f1c63a51340609bf4f963556da2627", + "zh:c9fa0106869bc9742b22e6afed0fe462a100d259013fa49b57414159f3acf0d4", + "zh:cce2a6ba1aff2157686ee7ba90f6c26acc09318f3047aefc830f50bb5a64e848", + "zh:e54c99c8e01f1ba2a1eda1364e2afbe15bce79c090c8f6f8f971775684ebb40d", + "zh:fdb533a8765072f6b499e9aedaaf84f45d4c0256861b960d0b3a3d651b1cad0a", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.12.0" + constraints = "~> 5.12.0" + hashes = [ + "h1:i28TUsgqoKs891cyDU0V9fFAwEz/RqbwF8sQShLfNq0=", + "zh:0953565eb67ece49556dc9046c77322dc6c76e0ae6fa0c9fd6710b6afa2588c9", + "zh:43676f3592c127a971719cc37b9199967376fb05d445b356f1545609e2b84bf8", + "zh:46422ab8044b35e90f422ffabc17fa043ec8e4a33e3df2f8b305d63a950c0edb", + "zh:4d34f024a82d31d10b5a9498d26fca71e3e35c543dfc5185c94c3205bc4dba22", + "zh:51be0eeb882f041fc2679bd621e64cd775d013ae003055cea013c9d630c15dfb", + "zh:7ca9252befa7271899febde25b679a73f90dbdb700cbbfec07d29389a3937131", + "zh:8325b2152be0534a718e497a3273cf6c42880e78f290dc35024feef2e0af8e97", + "zh:98f0c4d4c190cf4897cb9075a538f42f2998566e9f2d15755901fbb4862f8b32", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a71e0bc6754bb3924b31727d80f05b04fa65247c009ffdfd2a715a01d95b373d", + "zh:a82ae67ce3d4c7aaae761a592275b8cac5e9965a30b2dba951c1d965b3121006", + "zh:c5510eca023cec89557a8244648bf8ad9a0cd3189b6abf6dcceba30e3b2e8c6d", + "zh:cd11fe9c83793e838b6f90a55840fc45e7c106b358a68f0a88db09a29a321c9a", + "zh:e451ad353f219a2922b92e786a93c31658168b896317be127798cddfa9a99363", + "zh:e4b70a70e925b9ccb7d44e17fd8e7b89aa744a965f298f8bb2480a5c96f3c4f0", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.3.2" + constraints = "~> 3.3.2" + hashes = [ + "h1:YChjos7Hrvr2KgTc9GzQ+de/QE2VLAeRJgxFemnCltU=", + "zh:038293aebfede983e45ee55c328e3fde82ae2e5719c9bd233c324cfacc437f9c", + "zh:07eaeab03a723d83ac1cc218f3a59fceb7bbf301b38e89a26807d1c93c81cef8", + "zh:427611a4ce9d856b1c73bea986d841a969e4c2799c8ac7c18798d0cc42b78d32", + "zh:49718d2da653c06a70ba81fd055e2b99dfd52dcb86820a6aeea620df22cd3b30", + "zh:5574828d90b19ab762604c6306337e6cd430e65868e13ef6ddb4e25ddb9ad4c0", + "zh:7222e16f7833199dabf1bc5401c56d708ec052b2a5870988bc89ff85b68a5388", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b1b2d7d934784d2aee98b0f8f07a8ccfc0410de63493ae2bf2222c165becf938", + "zh:b8f85b6a20bd264fcd0814866f415f0a368d1123cd7879c8ebbf905d370babc8", + "zh:c3813133acc02bbebddf046d9942e8ba5c35fc99191e3eb057957dafc2929912", + "zh:e7a41dbc919d1de800689a81c240c27eec6b9395564630764ebb323ea82ac8a9", + "zh:ee6d23208449a8eaa6c4f203e33f5176fa795b4b9ecf32903dffe6e2574732c2", + ] +} diff --git a/iac/terraform/aws-services/backend.tf b/iac/terraform/aws-services/backend.tf new file mode 100644 index 0000000..bd30223 --- /dev/null +++ b/iac/terraform/aws-services/backend.tf @@ -0,0 +1,8 @@ +terraform { + backend "s3" { + region = "us-west-2" + key = "aws-services" + workspace_key_prefix = "tenant:" + encrypt = true + } +} diff --git a/iac/terraform/aws-services/efs.tf b/iac/terraform/aws-services/efs.tf new file mode 100644 index 0000000..316c64f --- /dev/null +++ b/iac/terraform/aws-services/efs.tf @@ -0,0 +1,42 @@ + +resource "duplocloud_aws_efs_file_system" "efs" { + backup = false + encrypted = true + name = "storage-${local.tenant_name}" + performance_mode = "generalPurpose" + tenant_id = local.tenant_id + throughput_mode = "bursting" +} + +resource "duplocloud_k8_storage_class" "sc" { + tenant_id = local.tenant_id + name = "storage" + storage_provisioner = "efs.csi.aws.com" + reclaim_policy = "Retain" + volume_binding_mode = "Immediate" + allow_volume_expansion = true + parameters = { + fileSystemId = duplocloud_aws_efs_file_system.efs.file_system_id + basePath = "/dynamic_provisioning", + directoryPerms = "700", + uid = "1000", + gid = "2000", + provisioningMode = "efs-ap", + } +} + +resource "duplocloud_k8_persistent_volume_claim" "pvc" { + + tenant_id = local.tenant_id + name = "storage" + spec { + access_modes = ["ReadWriteMany"] + volume_mode = "Filesystem" + storage_class_name = duplocloud_k8_storage_class.sc.fullname + resources { + requests = { + storage = "100Gi" + } + } + } +} diff --git a/iac/terraform/aws-services/hosts.tf b/iac/terraform/aws-services/hosts.tf new file mode 100644 index 0000000..91ac789 --- /dev/null +++ b/iac/terraform/aws-services/hosts.tf @@ -0,0 +1,46 @@ + +data "duplocloud_native_host_image" "ami" { + tenant_id = data.terraform_remote_state.tenant.outputs.tenant_id + is_kubernetes = true +} + +# in non-prod environments we will use a single instance, this will allow us to easily use a scheduler to shutdown the instance durring off hours +resource "duplocloud_aws_host" "hosts" { + + count = terraform.workspace == "ask-prod" ? 0 : 1 + + tenant_id = local.tenant_id + friendly_name = "host" + image_id = data.duplocloud_native_host_image.ami.image_id + capacity = var.asg_instance_type + agent_platform = 7 + zone = 1 + is_minion = true + is_ebs_optimized = false + encrypt_disk = false + allocated_public_ip = false + cloud = 0 + keypair_type = 2 + + metadata { + key = "OsDiskSize" + value = var.asg_os_disk_size + } +} + +module "nodegroup" { + + count = terraform.workspace == "ask-prod" ? 1 : 0 + + source = "duplocloud/components/duplocloud//modules/eks-nodes" + version = "0.0.5" + tenant_id = local.tenant_id + os_disk_size = var.asg_os_disk_size + eks_version = data.duplocloud_plan.plan.kubernetes_config[0]["version"] + + az_list = var.az_list + capacity = var.asg_instance_type + instance_count = var.asg_instance_count + min_instance_count = var.asg_instance_count + max_instance_count = var.asg_instance_count +} diff --git a/iac/terraform/aws-services/main.tf b/iac/terraform/aws-services/main.tf new file mode 100644 index 0000000..729f251 --- /dev/null +++ b/iac/terraform/aws-services/main.tf @@ -0,0 +1,45 @@ +data "aws_caller_identity" "current" { +} + +data "duplocloud_aws_account" "aws" { + tenant_id = local.tenant_name +} + +data "duplocloud_tenant" "tenant" { + name = local.tenant_name +} + + +data "duplocloud_tenant_aws_region" "current" { + tenant_id = local.tenant_id +} + +locals { + tfstate_bucket = "duplo-tfstate-${data.aws_caller_identity.current.account_id}" + region = data.duplocloud_tenant_aws_region.current.aws_region + tenant_id = data.duplocloud_tenant.tenant.id + cert_arn = data.terraform_remote_state.tenant.outputs["cert_arn"] + tenant_name = terraform.workspace + infra_name = data.terraform_remote_state.tenant.outputs["infra_name"] +} + +data "duplocloud_tenant_aws_kms_key" "tenant_kms" { + tenant_id = local.tenant_id +} + +data "duplocloud_plan" "plan" { + plan_id = local.infra_name +} + +data "terraform_remote_state" "tenant" { + backend = "s3" + workspace = terraform.workspace + config = { + bucket = local.tfstate_bucket + workspace_key_prefix = "admin:" + key = "tenant" + region = "us-west-2" + } +} + + diff --git a/iac/terraform/aws-services/outputs.tf b/iac/terraform/aws-services/outputs.tf new file mode 100644 index 0000000..70a7022 --- /dev/null +++ b/iac/terraform/aws-services/outputs.tf @@ -0,0 +1,29 @@ + +output "s3_search_fullname" { + value = duplocloud_s3_bucket.search.fullname + description = "The full name of the S3 bucket." +} +output "s3_search_arn" { + value = duplocloud_s3_bucket.search.arn + description = "The ARN of the S3 bucket." +} + +output "sqs_queue_fullname" { + value = duplocloud_aws_sqs_queue.sqs_queue.fullname + description = "Name of the SQS queue" +} + +output "sqs_queue_arn" { + value = duplocloud_aws_sqs_queue.sqs_queue.arn + description = "ARN of the SQS queue" +} + +output "sqs_queue_url" { + value = duplocloud_aws_sqs_queue.sqs_queue.url + description = "URL of the SQS queue" +} + +output "region" { + value = local.region + description = "The AWS Region for the Tenant" +} diff --git a/iac/terraform/aws-services/providers.tf b/iac/terraform/aws-services/providers.tf new file mode 100644 index 0000000..c2f8579 --- /dev/null +++ b/iac/terraform/aws-services/providers.tf @@ -0,0 +1,21 @@ +terraform { + required_version = ">= 1.5.5" + required_providers { + duplocloud = { + source = "duplocloud/duplocloud" + version = "~> 0.10.2" + } + random = { + source = "hashicorp/random" + version = "~> 3.3.2" + } + } +} +provider "duplocloud" { + +} +provider "aws" { + region = data.duplocloud_tenant_aws_region.current.aws_region +} + +provider "random" {} diff --git a/iac/terraform/aws-services/s3.tf b/iac/terraform/aws-services/s3.tf new file mode 100644 index 0000000..4fa8c0e --- /dev/null +++ b/iac/terraform/aws-services/s3.tf @@ -0,0 +1,12 @@ + +resource "duplocloud_s3_bucket" "search" { + tenant_id = local.tenant_id + name = "search" + allow_public_access = var.s3_search_allow_public_access + enable_access_logs = var.s3_search_enable_access_logs + enable_versioning = var.s3_search_enable_versioning + managed_policies = ["ssl"] + default_encryption { + method = "Sse" + } +} diff --git a/iac/terraform/aws-services/sqs.tf b/iac/terraform/aws-services/sqs.tf new file mode 100644 index 0000000..73c4391 --- /dev/null +++ b/iac/terraform/aws-services/sqs.tf @@ -0,0 +1,54 @@ + +resource "duplocloud_aws_sqs_queue" "sqs_queue" { + tenant_id = local.tenant_id + name = "database-updates" + fifo_queue = false + message_retention_seconds = 345600 + visibility_timeout_seconds = 30 +} + +data "aws_iam_policy_document" "queue" { + statement { + sid = "example-statement-ID" + effect = "Allow" + resources = [duplocloud_aws_sqs_queue.sqs_queue.arn] + actions = ["SQS:SendMessage" ] + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [duplocloud_s3_bucket.search.arn] + } + + condition { + test = "StringEquals" + variable = "aws:SourceAccount" + values = [data.duplocloud_aws_account.aws.account_id] + } + + principals { + type = "Service" + identifiers = ["s3.amazonaws.com"] + } + } +} + +resource "aws_sqs_queue_policy" "policy" { + queue_url = duplocloud_aws_sqs_queue.sqs_queue.url + policy = data.aws_iam_policy_document.queue.json +} + + +resource "aws_s3_bucket_notification" "bucket_notification" { + + depends_on = [ aws_sqs_queue_policy.policy ] + + bucket = duplocloud_s3_bucket.search.fullname + + queue { + id = "sqs" + queue_arn = duplocloud_aws_sqs_queue.sqs_queue.arn + events = ["s3:ObjectCreated:*", "s3:ObjectRemoved:*"] + filter_prefix = "source_documents/" + } +} diff --git a/iac/terraform/aws-services/vars.tf b/iac/terraform/aws-services/vars.tf new file mode 100644 index 0000000..691074a --- /dev/null +++ b/iac/terraform/aws-services/vars.tf @@ -0,0 +1,35 @@ + +variable "asg_instance_type" { + default = "t3.xlarge" + type = string +} +variable "s3_search_allow_public_access" { + default = false + type = bool +} +variable "s3_search_enable_access_logs" { + default = false + type = bool +} +variable "s3_search_enable_versioning" { + default = true + type = bool +} + +variable "asg_instance_count" { + default = 1 + description = "Number of nodes in each ASG (one ASG per AZ)" + type = number +} + +variable "asg_os_disk_size" { + default = 100 + type = string +} + +# default to a single ASG so that all non production environments will have single node +variable "az_list" { + default = ["a"] + type = list(string) + description = "The letter at the end of the zone" +}