From 1015aad4888cd73a56489e0b86329504f0906aa0 Mon Sep 17 00:00:00 2001 From: Dmitry Gurevich <99176494+gurevichdmitry@users.noreply.github.com> Date: Wed, 26 Jul 2023 15:30:27 +0300 Subject: [PATCH] eks clusters migration (#1139) --- .github/workflows/eks-ci.yml | 10 ++++++---- .../test-eks-config-1-node-1-10-kubelet-args.conf | 2 +- .../test-eks-config-1-node-1-kubelet-config.json | 2 ++ .../test-eks-config-1-node-2-10-kubelet-args.conf | 2 +- .../test-eks-config-1-node-2-kubelet-config.json | 2 ++ .../test-eks-config-2-node-1-10-kubelet-args.conf | 2 +- .../test-eks-config-2-node-1-kubelet-config.json | 2 ++ .../test-eks-config-2-node-2-10-kubelet-args.conf | 2 +- .../test-eks-config-2-node-2-kubelet-config.json | 2 ++ .../templates/_eks-psp-failures-pod.yaml | 2 +- .../templates/_eks-psp-pass-pod.yaml | 2 +- .../templates/cloudbeat-eks-pytest.yaml | 3 ++- .../templates/cloudbeat-test-configmap.yml | 3 +++ .../k8s-cloudbeat-tests/templates/eks-results.yml | 2 +- tests/deploy/k8s-cloudbeat-tests/values.yaml | 7 +++++++ tests/deploy/values/ci-eks-config-1.yml | 2 +- tests/deploy/values/ci-eks-config-2.yml | 2 +- 17 files changed, 35 insertions(+), 14 deletions(-) diff --git a/.github/workflows/eks-ci.yml b/.github/workflows/eks-ci.yml index 4d90085c2c..349850637b 100644 --- a/.github/workflows/eks-ci.yml +++ b/.github/workflows/eks-ci.yml @@ -16,16 +16,16 @@ on: - "[0-9]+.[0-9]+" env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_TEST_ACC }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_TEST_ACC }} GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} CONTAINER_SUFFIX: ${{ github.run_id }} TEST_TARGETS_DEFAULT: '["pre_merge"]' AWS_REGION: eu-west-2 REPORTS_DIR: tests/allure/results/ KUBE_NAMESPACE: kube-system - CI_ELASTIC_AGENT_DOCKER_TAG: "8.7.0-SNAPSHOT" - CI_ELASTIC_AGENT_DOCKER_IMAGE: "704479110758.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent" + CI_ELASTIC_AGENT_DOCKER_TAG: "8.10.0-SNAPSHOT" + CI_ELASTIC_AGENT_DOCKER_IMAGE: "391946104644.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent" # run only a single job at a time concurrency: EKS-Functional-Tests @@ -250,6 +250,8 @@ jobs: env: GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} AWS_REGION: eu-west-1 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} with: storageType: s3 resultsGlob: "tests/allure/results" diff --git a/tests/deploy/aws/eks_config/test-eks-config-1-node-1-10-kubelet-args.conf b/tests/deploy/aws/eks_config/test-eks-config-1-node-1-10-kubelet-args.conf index d1452884b7..39c4faf00e 100644 --- a/tests/deploy/aws/eks_config/test-eks-config-1-node-1-10-kubelet-args.conf +++ b/tests/deploy/aws/eks_config/test-eks-config-1-node-1-10-kubelet-args.conf @@ -1,2 +1,2 @@ [Service] -Environment='KUBELET_ARGS=--node-ip=192.168.15.75 --pod-infra-container-image=602401143452.dkr.ecr.eu-west-2.amazonaws.com/eks/pause:3.5 --v=2 --cloud-provider=aws --read-only-port=26492 --streaming-connection-idle-timeout=0' +Environment='KUBELET_ARGS=--node-ip=192.168.29.162 --pod-infra-container-image=602401143452.dkr.ecr.eu-west-2.amazonaws.com/eks/pause:3.5 --v=2 --cloud-provider=aws --read-only-port=26492 --streaming-connection-idle-timeout=0' diff --git a/tests/deploy/aws/eks_config/test-eks-config-1-node-1-kubelet-config.json b/tests/deploy/aws/eks_config/test-eks-config-1-node-1-kubelet-config.json index 26725770c7..0d67351ad4 100644 --- a/tests/deploy/aws/eks_config/test-eks-config-1-node-1-kubelet-config.json +++ b/tests/deploy/aws/eks_config/test-eks-config-1-node-1-kubelet-config.json @@ -46,6 +46,8 @@ "clusterDNS": [ "10.100.0.10" ], + "kubeAPIQPS": 10, + "kubeAPIBurst": 20, "evictionHard": { "memory.available": "100Mi", "nodefs.available": "10%", diff --git a/tests/deploy/aws/eks_config/test-eks-config-1-node-2-10-kubelet-args.conf b/tests/deploy/aws/eks_config/test-eks-config-1-node-2-10-kubelet-args.conf index 6e469682eb..bc48398e61 100644 --- a/tests/deploy/aws/eks_config/test-eks-config-1-node-2-10-kubelet-args.conf +++ b/tests/deploy/aws/eks_config/test-eks-config-1-node-2-10-kubelet-args.conf @@ -1,2 +1,2 @@ [Service] -Environment='KUBELET_ARGS=--node-ip=192.168.38.87 --pod-infra-container-image=602401143452.dkr.ecr.eu-west-2.amazonaws.com/eks/pause:3.5 --v=2 --cloud-provider=aws --make-iptables-util-chains=false --event-qps=5 --hostname-override' +Environment='KUBELET_ARGS=--node-ip=192.168.93.227 --pod-infra-container-image=602401143452.dkr.ecr.eu-west-2.amazonaws.com/eks/pause:3.5 --v=2 --cloud-provider=aws --make-iptables-util-chains=false --event-qps=5 --hostname-override' diff --git a/tests/deploy/aws/eks_config/test-eks-config-1-node-2-kubelet-config.json b/tests/deploy/aws/eks_config/test-eks-config-1-node-2-kubelet-config.json index 32c173e685..0ee0f786bc 100644 --- a/tests/deploy/aws/eks_config/test-eks-config-1-node-2-kubelet-config.json +++ b/tests/deploy/aws/eks_config/test-eks-config-1-node-2-kubelet-config.json @@ -44,6 +44,8 @@ "clusterDNS": [ "10.100.0.10" ], + "kubeAPIQPS": 10, + "kubeAPIBurst": 20, "evictionHard": { "memory.available": "100Mi", "nodefs.available": "10%", diff --git a/tests/deploy/aws/eks_config/test-eks-config-2-node-1-10-kubelet-args.conf b/tests/deploy/aws/eks_config/test-eks-config-2-node-1-10-kubelet-args.conf index 9c7691ce37..b84bb5aa04 100644 --- a/tests/deploy/aws/eks_config/test-eks-config-2-node-1-10-kubelet-args.conf +++ b/tests/deploy/aws/eks_config/test-eks-config-2-node-1-10-kubelet-args.conf @@ -1,2 +1,2 @@ [Service] -Environment='KUBELET_ARGS=--node-ip=192.168.14.74 --pod-infra-container-image=602401143452.dkr.ecr.eu-west-2.amazonaws.com/eks/pause:3.5 --v=2 --streaming-connection-idle-timeout=26492s' +Environment='KUBELET_ARGS=--node-ip=192.168.33.106 --pod-infra-container-image=602401143452.dkr.ecr.eu-west-2.amazonaws.com/eks/pause:3.5 --v=2 --streaming-connection-idle-timeout=26492s' diff --git a/tests/deploy/aws/eks_config/test-eks-config-2-node-1-kubelet-config.json b/tests/deploy/aws/eks_config/test-eks-config-2-node-1-kubelet-config.json index 5993556daa..7b346dbf4a 100644 --- a/tests/deploy/aws/eks_config/test-eks-config-2-node-1-kubelet-config.json +++ b/tests/deploy/aws/eks_config/test-eks-config-2-node-1-kubelet-config.json @@ -45,6 +45,8 @@ "clusterDNS": [ "10.100.0.10" ], + "kubeAPIQPS": 10, + "kubeAPIBurst": 20, "evictionHard": { "memory.available": "100Mi", "nodefs.available": "10%", diff --git a/tests/deploy/aws/eks_config/test-eks-config-2-node-2-10-kubelet-args.conf b/tests/deploy/aws/eks_config/test-eks-config-2-node-2-10-kubelet-args.conf index 73b993e04b..4b0af9fa55 100644 --- a/tests/deploy/aws/eks_config/test-eks-config-2-node-2-10-kubelet-args.conf +++ b/tests/deploy/aws/eks_config/test-eks-config-2-node-2-10-kubelet-args.conf @@ -1,2 +1,2 @@ [Service] -Environment='KUBELET_ARGS=--node-ip=192.168.89.216 --pod-infra-container-image=602401143452.dkr.ecr.eu-west-2.amazonaws.com/eks/pause:3.5 --v=2 --read-only-port=0 --streaming-connection-idle-timeout=0 --make-iptables-util-chains=false --event-qps=0' +Environment='KUBELET_ARGS=--node-ip=192.168.6.157 --pod-infra-container-image=602401143452.dkr.ecr.eu-west-2.amazonaws.com/eks/pause:3.5 --v=2 --read-only-port=0 --streaming-connection-idle-timeout=0 --make-iptables-util-chains=false --event-qps=0' diff --git a/tests/deploy/aws/eks_config/test-eks-config-2-node-2-kubelet-config.json b/tests/deploy/aws/eks_config/test-eks-config-2-node-2-kubelet-config.json index e7faf53d34..ddc1c00711 100644 --- a/tests/deploy/aws/eks_config/test-eks-config-2-node-2-kubelet-config.json +++ b/tests/deploy/aws/eks_config/test-eks-config-2-node-2-kubelet-config.json @@ -47,6 +47,8 @@ "clusterDNS": [ "10.100.0.10" ], + "kubeAPIQPS": 10, + "kubeAPIBurst": 20, "evictionHard": { "memory.available": "100Mi", "nodefs.available": "10%", diff --git a/tests/deploy/k8s-cloudbeat-tests/templates/_eks-psp-failures-pod.yaml b/tests/deploy/k8s-cloudbeat-tests/templates/_eks-psp-failures-pod.yaml index 40d6bd7434..e64b851914 100644 --- a/tests/deploy/k8s-cloudbeat-tests/templates/_eks-psp-failures-pod.yaml +++ b/tests/deploy/k8s-cloudbeat-tests/templates/_eks-psp-failures-pod.yaml @@ -13,7 +13,7 @@ spec: automountServiceAccountToken: false containers: - name: busybox - image: 704479110758.dkr.ecr.eu-west-2.amazonaws.com/test-eks-scan-false:latest + image: 391946104644.dkr.ecr.eu-west-2.amazonaws.com/test-eks-scan-false:latest imagePullPolicy: Always command: - sleep diff --git a/tests/deploy/k8s-cloudbeat-tests/templates/_eks-psp-pass-pod.yaml b/tests/deploy/k8s-cloudbeat-tests/templates/_eks-psp-pass-pod.yaml index 3c7e895da8..7e8452cc2a 100644 --- a/tests/deploy/k8s-cloudbeat-tests/templates/_eks-psp-pass-pod.yaml +++ b/tests/deploy/k8s-cloudbeat-tests/templates/_eks-psp-pass-pod.yaml @@ -13,7 +13,7 @@ spec: automountServiceAccountToken: false containers: - name: busybox - image: 704479110758.dkr.ecr.eu-west-2.amazonaws.com/test-eks-scan-true:latest + image: 391946104644.dkr.ecr.eu-west-2.amazonaws.com/test-eks-scan-true:latest imagePullPolicy: Always command: - sleep diff --git a/tests/deploy/k8s-cloudbeat-tests/templates/cloudbeat-eks-pytest.yaml b/tests/deploy/k8s-cloudbeat-tests/templates/cloudbeat-eks-pytest.yaml index fb10fb0941..a4fb2dbfc0 100644 --- a/tests/deploy/k8s-cloudbeat-tests/templates/cloudbeat-eks-pytest.yaml +++ b/tests/deploy/k8s-cloudbeat-tests/templates/cloudbeat-eks-pytest.yaml @@ -291,7 +291,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: 704479110758.dkr.ecr.eu-west-2.amazonaws.com/cloudbeat:latest + image: "{{ .Values.cloudbeat.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{.Values.image.pullPolicy}} name: cloudbeat resources: requests: diff --git a/tests/deploy/k8s-cloudbeat-tests/templates/cloudbeat-test-configmap.yml b/tests/deploy/k8s-cloudbeat-tests/templates/cloudbeat-test-configmap.yml index 80354851ae..80b798584b 100644 --- a/tests/deploy/k8s-cloudbeat-tests/templates/cloudbeat-test-configmap.yml +++ b/tests/deploy/k8s-cloudbeat-tests/templates/cloudbeat-test-configmap.yml @@ -10,6 +10,9 @@ data: ES_PASSWORD: "changeme" ES_HOST: "elasticsearch-master.kube-system" USE_DOCKER: "false" + {{- range $key, $value := .Values.testData.eksNodes }} + {{ $key }}: {{ $value }} + {{- end }} MARKER_CMD: {{ .Values.testData.markerCmd}} TEST_MARKER: {{ .Values.testData.marker }} {{- if .Values.testData.range }} diff --git a/tests/deploy/k8s-cloudbeat-tests/templates/eks-results.yml b/tests/deploy/k8s-cloudbeat-tests/templates/eks-results.yml index 450f22938a..9f384fa1e8 100644 --- a/tests/deploy/k8s-cloudbeat-tests/templates/eks-results.yml +++ b/tests/deploy/k8s-cloudbeat-tests/templates/eks-results.yml @@ -15,7 +15,7 @@ spec: spec: containers: - name: busybox - image: 704479110758.dkr.ecr.eu-west-2.amazonaws.com/test-eks-scan-true:latest + image: "{{ .Values.image.ecr }}/test-eks-scan-true:latest" imagePullPolicy: IfNotPresent command: - sleep diff --git a/tests/deploy/k8s-cloudbeat-tests/values.yaml b/tests/deploy/k8s-cloudbeat-tests/values.yaml index 40f354fc7d..83862108be 100644 --- a/tests/deploy/k8s-cloudbeat-tests/values.yaml +++ b/tests/deploy/k8s-cloudbeat-tests/values.yaml @@ -10,12 +10,18 @@ image: pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "latest" + ecr: 391946104644.dkr.ecr.eu-west-2.amazonaws.com testData: marker: rules agentName: cloudbeat markerCmd: -m clusterConfig: test-eks-config-1 + eksNodes: + EKS_CONFIG_1_NODE_1: "ip-192-168-29-162.eu-west-2.compute.internal" + EKS_CONFIG_1_NODE_2: "ip-192-168-93-227.eu-west-2.compute.internal" + EKS_CONFIG_2_NODE_1: "ip-192-168-33-106.eu-west-2.compute.internal" + EKS_CONFIG_2_NODE_2: "ip-192-168-6-157.eu-west-2.compute.internal" imagePullSecrets: [] nameOverride: "" @@ -99,6 +105,7 @@ job: cloudbeat: deploy: false + repository: 391946104644.dkr.ecr.eu-west-2.amazonaws.com/cloudbeat cloudbeatEks: deploy: false diff --git a/tests/deploy/values/ci-eks-config-1.yml b/tests/deploy/values/ci-eks-config-1.yml index 3fa538eb82..a7d234f6d9 100644 --- a/tests/deploy/values/ci-eks-config-1.yml +++ b/tests/deploy/values/ci-eks-config-1.yml @@ -3,7 +3,7 @@ elasticsearch: storageClassName: gp2 image: - repository: 704479110758.dkr.ecr.eu-west-2.amazonaws.com/cloudbeat-test + repository: 391946104644.dkr.ecr.eu-west-2.amazonaws.com/cloudbeat-test pullPolicy: Always cloudbeat: diff --git a/tests/deploy/values/ci-eks-config-2.yml b/tests/deploy/values/ci-eks-config-2.yml index a0a97b378d..73122729b3 100644 --- a/tests/deploy/values/ci-eks-config-2.yml +++ b/tests/deploy/values/ci-eks-config-2.yml @@ -3,7 +3,7 @@ elasticsearch: storageClassName: gp2 image: - repository: 704479110758.dkr.ecr.eu-west-2.amazonaws.com/cloudbeat-test + repository: 391946104644.dkr.ecr.eu-west-2.amazonaws.com/cloudbeat-test pullPolicy: Always cloudbeat: