From 56aeb4c1561f08bdd55f16a5ead77a28b0ee221a Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Wed, 18 Oct 2023 16:47:38 +1100 Subject: [PATCH 01/19] Don't make commits automatically in `make release` I prefer to make the commits myself thank you very much. --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index fac9b6b53..1549c4893 100644 --- a/Makefile +++ b/Makefile @@ -197,8 +197,6 @@ bump-agent-version: sed -i.bak -E "s/AGENT_VERSION=.+/AGENT_VERSION=$(AGENT_VERSION)/g" packer/linux/scripts/install-buildkite-agent.sh sed -i.bak -E "s/\\\$$AGENT_VERSION = \".+\"/\$$AGENT_VERSION = \"$(AGENT_VERSION)\"/g" packer/windows/scripts/install-buildkite-agent.ps1 rm README.md.bak packer/linux/scripts/install-buildkite-agent.sh.bak packer/windows/scripts/install-buildkite-agent.ps1.bak - git add README.md packer/linux/scripts/install-buildkite-agent.sh packer/windows/scripts/install-buildkite-agent.ps1 - git commit -m "Bump buildkite-agent to v$(AGENT_VERSION)" validate: build/aws-stack.yml aws --no-cli-pager cloudformation validate-template \ From 42467d722d2b0629018c782e12653b5ab270ca76 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Wed, 18 Oct 2023 23:18:44 +1100 Subject: [PATCH 02/19] Tag AMI with agent version --- Makefile | 5 +++++ packer/linux/buildkite-ami.pkr.hcl | 6 ++++++ packer/windows/buildkite-ami.pkr.hcl | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/Makefile b/Makefile index 1549c4893..6728a2eac 100644 --- a/Makefile +++ b/Makefile @@ -111,6 +111,9 @@ build/linux-arm64-ami.txt: packer-linux-arm64.output env-AWS_REGION mkdir -p build grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@ +CURRENT_AGENT_VERSION_LINUX ?= $(shell sed -En 's/^AGENT_VERSION="?(.+?)"?$/\1/p' packer/linux/scripts/install-buildkite-agent.sh) +CURRENT_AGENT_VERSION_WINDOWS ?= $(shell sed -En 's/^\$AGENT_VERSION = "(.+?)"$/\1/p' packer/windows/scripts/install-buildkite-agent.ps1) + # Build linuxarm64 packer image packer-linux-arm64.output: $(PACKER_LINUX_FILES) docker run \ @@ -130,6 +133,7 @@ packer-linux-arm64.output: $(PACKER_LINUX_FILES) -var 'instance_type=$(ARM64_INSTANCE_TYPE)' \ -var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \ -var 'is_released=$(IS_RELEASED)' \ + -var 'agent_version=$(CURRENT_AGENT_VERSION_LINUX)' \ buildkite-ami.pkr.hcl | tee $@ build/windows-amd64-ami.txt: packer-windows-amd64.output env-AWS_REGION @@ -155,6 +159,7 @@ packer-windows-amd64.output: $(PACKER_WINDOWS_FILES) -var 'instance_type=$(WIN64_INSTANCE_TYPE)' \ -var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \ -var 'is_released=$(IS_RELEASED)' \ + -var 'agent_version=$(CURRENT_AGENT_VERSION_WINDOWS)' \ buildkite-ami.pkr.hcl | tee $@ # ----------------------------------------- diff --git a/packer/linux/buildkite-ami.pkr.hcl b/packer/linux/buildkite-ami.pkr.hcl index 4eee24282..ac7c77c4d 100644 --- a/packer/linux/buildkite-ami.pkr.hcl +++ b/packer/linux/buildkite-ami.pkr.hcl @@ -27,6 +27,11 @@ variable "build_number" { default = "none" } +variable "agent_version" { + type = string + default = "devel" +} + variable "is_released" { type = bool default = false @@ -56,6 +61,7 @@ source "amazon-ebs" "elastic-ci-stack-ami" { Name = "elastic-ci-stack-linux-${var.arch}" OSVersion = "Amazon Linux 2023" BuildNumber = var.build_number + AgentVersion = var.agent_version IsReleased = var.is_released SourceAMIID = data.amazon-ami.al2023.id SourceAMIName = data.amazon-ami.al2023.name diff --git a/packer/windows/buildkite-ami.pkr.hcl b/packer/windows/buildkite-ami.pkr.hcl index 1fe5e490a..609f345b3 100644 --- a/packer/windows/buildkite-ami.pkr.hcl +++ b/packer/windows/buildkite-ami.pkr.hcl @@ -27,6 +27,11 @@ variable "build_number" { default = "none" } +variable "agent_version" { + type = string + default = "devel" +} + variable "is_released" { type = bool default = false @@ -59,6 +64,7 @@ source "amazon-ebs" "elastic-ci-stack" { Name = "elastic-ci-stack-windows" OSVersion = "Windows Server 2019" BuildNumber = var.build_number + AgentVersion = var.agent_version IsReleased = var.is_released SourceAMIID = data.amazon-ami.windows-server-2019.id SourceAMIName = data.amazon-ami.windows-server-2019.name From 1ad59336f55addf27a151a1790b9242be03fb7c2 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Wed, 18 Oct 2023 23:21:19 +1100 Subject: [PATCH 03/19] Propogate tags when copying to other regions --- .buildkite/steps/copy.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index ed38d2585..4a7f275f9 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -11,11 +11,12 @@ copy_ami_to_region() { local destination_image_name="$4" aws ec2 copy-image \ + --copy-image-tags \ --source-image-id "$source_image_id" \ --source-region "$source_region" \ --name "$destination_image_name" \ --region "$destination_image_region" \ - --query "ImageId" \ + --query ImageId \ --output text } From b1c38e630b7c10cfe70bfece6aa1022a2e55759b Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 07:22:43 +1100 Subject: [PATCH 04/19] Tag images with IsReleased when they are released --- .buildkite/steps/copy.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index 4a7f275f9..23da744c7 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -118,6 +118,10 @@ EOF exit 0 fi +aws ec2 tag-images --region "$source_region" --image-ids "$linux_amd64_source_image_id" --tags Key=IsReleased,Value=true +aws ec2 tag-images --region "$source_region" --image-ids "$linux_arm64_source_image_id" --tags Key=IsReleased,Value=true +aws ec2 tag-images --region "$source_region" --image-ids "$windows_amd64_source_image_id" --tags Key=IsReleased,Value=true + s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s-%s.yml" \ "${BUILDKITE_AWS_STACK_BUCKET}" \ "${linux_amd64_source_image_id}" \ From 7f5057c4b5f19c8b24a3abcb793f0ecddc258f57 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 07:28:43 +1100 Subject: [PATCH 05/19] Heed SC2048 The regions don't have spaces in time, so this should be a no-op. I prefer to write code that complies with the linter if its just a few extra characters. --- .buildkite/steps/copy.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index 23da744c7..cd006def7 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -141,8 +141,7 @@ linux_arm64_source_image_name=$(get_image_name "$linux_arm64_source_image_id" "$ windows_amd64_source_image_name=$(get_image_name "$windows_amd64_source_image_id" "$source_region") # Copy to all other regions -# shellcheck disable=SC2048 -for region in ${ALL_REGIONS[*]}; do +for region in "${ALL_REGIONS[@]}"; do if [[ $region != "$source_region" ]]; then echo "--- :linux: Copying Linux AMD64 $linux_amd64_source_image_id to $region" >&2 IMAGES+=("$(copy_ami_to_region "$linux_amd64_source_image_id" "$source_region" "$region" "${linux_amd64_source_image_name}-${region}")") @@ -165,8 +164,7 @@ Mappings: EOF echo "--- Waiting for AMIs to become available" >&2 -# shellcheck disable=SC2048 -for region in ${ALL_REGIONS[*]}; do +for region in "${ALL_REGIONS[@]}"; do linux_amd64_image_id="${IMAGES[0]}" linux_arm64_image_id="${IMAGES[1]}" windows_amd64_image_id="${IMAGES[2]}" From 26403d46128b5452af4cb0c1049cb4cc488e8624 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 11:15:26 +1100 Subject: [PATCH 06/19] Add some informative log groups --- .buildkite/steps/copy.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index cd006def7..0856b734e 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -118,10 +118,12 @@ EOF exit 0 fi +echo --- Tagging source images as released aws ec2 tag-images --region "$source_region" --image-ids "$linux_amd64_source_image_id" --tags Key=IsReleased,Value=true aws ec2 tag-images --region "$source_region" --image-ids "$linux_arm64_source_image_id" --tags Key=IsReleased,Value=true aws ec2 tag-images --region "$source_region" --image-ids "$windows_amd64_source_image_id" --tags Key=IsReleased,Value=true +echo --- Checking if there is a previously copy in the cache bucket s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s-%s.yml" \ "${BUILDKITE_AWS_STACK_BUCKET}" \ "${linux_amd64_source_image_id}" \ @@ -129,12 +131,13 @@ s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s-%s.yml" \ "${windows_amd64_source_image_id}" \ "${BUILDKITE_BRANCH}") -# Check if there is a previously copy in the cache bucket if aws s3 cp "${s3_mappings_cache}" "$mapping_file"; then echo "--- Skipping AMI copy, was previously copied" exit 0 fi +echo --- Copying images to other regions + # Get the image names to copy to other regions linux_amd64_source_image_name=$(get_image_name "$linux_amd64_source_image_id" "$source_region") linux_arm64_source_image_name=$(get_image_name "$linux_arm64_source_image_id" "$source_region") From c16805512a267b3b2ae3c27c6017f850d6865ede Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 11:15:42 +1100 Subject: [PATCH 07/19] Create an independent environment variable to tag is_released --- .buildkite/steps/copy.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index 0856b734e..dd98a57dc 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -119,9 +119,11 @@ EOF fi echo --- Tagging source images as released -aws ec2 tag-images --region "$source_region" --image-ids "$linux_amd64_source_image_id" --tags Key=IsReleased,Value=true -aws ec2 tag-images --region "$source_region" --image-ids "$linux_arm64_source_image_id" --tags Key=IsReleased,Value=true -aws ec2 tag-images --region "$source_region" --image-ids "$windows_amd64_source_image_id" --tags Key=IsReleased,Value=true +if [[ $BUILDKITE_BRANCH == main || $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_IS_RELEASED:-false} == true ]]; then + aws ec2 tag-images --region "$source_region" --image-ids "$linux_amd64_source_image_id" --tags Key=IsReleased,Value=true + aws ec2 tag-images --region "$source_region" --image-ids "$linux_arm64_source_image_id" --tags Key=IsReleased,Value=true + aws ec2 tag-images --region "$source_region" --image-ids "$windows_amd64_source_image_id" --tags Key=IsReleased,Value=true +fi echo --- Checking if there is a previously copy in the cache bucket s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s-%s.yml" \ From bb261c02e0fe025d77b64aa30bd1dd4aa92c8218 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 07:31:50 +1100 Subject: [PATCH 08/19] Use binary operations inside [[ It reduces the length of this already very long condition. --- .buildkite/steps/copy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index dd98a57dc..7f0924374 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -107,7 +107,7 @@ if [ $# -eq 0 ]; then fi # If we're not on the main branch or a tag build skip the copy -if [[ $BUILDKITE_BRANCH != main ]] && [[ $BUILDKITE_TAG != "$BUILDKITE_BRANCH" ]] && [[ ${COPY_TO_ALL_REGIONS:-"false"} != "true" ]]; then +if [[ $BUILDKITE_BRANCH != main && $BUILDKITE_TAG != "$BUILDKITE_BRANCH" && ${COPY_TO_ALL_REGIONS:-false} != true ]]; then echo "--- Skipping AMI copy on non-main/tag branch " >&2 mkdir -p "$(dirname "$mapping_file")" cat <"$mapping_file" From 6ba2d404f0c21b31a788b1589218b7e5a165b19d Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 07:40:47 +1100 Subject: [PATCH 09/19] Fix escaping for shell commands --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6728a2eac..5ce6f32f0 100644 --- a/Makefile +++ b/Makefile @@ -111,8 +111,9 @@ build/linux-arm64-ami.txt: packer-linux-arm64.output env-AWS_REGION mkdir -p build grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@ -CURRENT_AGENT_VERSION_LINUX ?= $(shell sed -En 's/^AGENT_VERSION="?(.+?)"?$/\1/p' packer/linux/scripts/install-buildkite-agent.sh) -CURRENT_AGENT_VERSION_WINDOWS ?= $(shell sed -En 's/^\$AGENT_VERSION = "(.+?)"$/\1/p' packer/windows/scripts/install-buildkite-agent.ps1) +# NOTE: make removes the $ escapes, everything else is passed to the shell +CURRENT_AGENT_VERSION_LINUX ?= $(shell sed -En 's/^AGENT_VERSION="?(.+?)"?$$/\1/p' packer/linux/scripts/install-buildkite-agent.sh) +CURRENT_AGENT_VERSION_WINDOWS ?= $(shell sed -En 's/^\$$AGENT_VERSION = "(.+?)"$$/\1/p' packer/windows/scripts/install-buildkite-agent.ps1) # Build linuxarm64 packer image packer-linux-arm64.output: $(PACKER_LINUX_FILES) From 860931abcaf76b647602741cf3272613575b7cb9 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 08:30:36 +1100 Subject: [PATCH 10/19] Add emoji to the cleanup step --- .buildkite/pipeline.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index d22ccb23f..c483ebb12 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -212,8 +212,8 @@ steps: artifact_paths: "build/*.yml" depends_on: "copy-ami" - - id: "cleanup" - name: "Cleanup" + - id: cleanup + name: ":broom: Cleanup" command: .buildkite/steps/cleanup.sh agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}" From ccce448da74c7254d2c91b77097101179aedc1d7 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 11:13:55 +1100 Subject: [PATCH 11/19] Tag Version --- .buildkite/steps/copy.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index 7f0924374..22c18d6d2 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -125,6 +125,14 @@ if [[ $BUILDKITE_BRANCH == main || $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TA aws ec2 tag-images --region "$source_region" --image-ids "$windows_amd64_source_image_id" --tags Key=IsReleased,Value=true fi +echo --- Tagging elastic ci stack release version +echo Note: the same AMI may be used in multiple versions of the elastic stack, so we can\'t use the same tag key +if [[ $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_VERSION:-false} == true ]]; then + aws ec2 tag-images --region "$source_region" --image-ids "$linux_amd64_source_image_id" --tags "Key=Version:${BUILDKITE_TAG},Value=true" + aws ec2 tag-images --region "$source_region" --image-ids "$linux_arm64_source_image_id" --tags "Key=Version:${BUILDKITE_TAG},Value=true" + aws ec2 tag-images --region "$source_region" --image-ids "$windows_amd64_source_image_id" --tags "Key=Version:${BUILDKITE_TAG},Value=true" +fi + echo --- Checking if there is a previously copy in the cache bucket s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s-%s.yml" \ "${BUILDKITE_AWS_STACK_BUCKET}" \ From eae7875a850fe3607ccf890bdeb40d5202967dea Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 11:34:29 +1100 Subject: [PATCH 12/19] Modify agent version bump recipe to stop creating sed backups when they are deleted anyway --- Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 5ce6f32f0..2453b7425 100644 --- a/Makefile +++ b/Makefile @@ -199,10 +199,9 @@ update-stack: build/aws-stack.yml env-STACK_NAME AGENT_VERSION ?= $(shell curl -Lfs "https://buildkite.com/agent/releases/latest?platform=linux&arch=amd64" | grep version | cut -d= -f2) bump-agent-version: - sed -i.bak -E "s/\[Buildkite Agent v.*\]/[Buildkite Agent v$(AGENT_VERSION)]/g" README.md - sed -i.bak -E "s/AGENT_VERSION=.+/AGENT_VERSION=$(AGENT_VERSION)/g" packer/linux/scripts/install-buildkite-agent.sh - sed -i.bak -E "s/\\\$$AGENT_VERSION = \".+\"/\$$AGENT_VERSION = \"$(AGENT_VERSION)\"/g" packer/windows/scripts/install-buildkite-agent.ps1 - rm README.md.bak packer/linux/scripts/install-buildkite-agent.sh.bak packer/windows/scripts/install-buildkite-agent.ps1.bak + sed -iE "s/\[Buildkite Agent v.*\]/[Buildkite Agent v$(AGENT_VERSION)]/g" README.md + sed -iE "s/AGENT_VERSION=.+/AGENT_VERSION=$(AGENT_VERSION)/g" packer/linux/scripts/install-buildkite-agent.sh + sed -iE "s/\\\$$AGENT_VERSION = \".+\"/\$$AGENT_VERSION = \"$(AGENT_VERSION)\"/g" packer/windows/scripts/install-buildkite-agent.ps1 validate: build/aws-stack.yml aws --no-cli-pager cloudformation validate-template \ From 33228703799b07eb946005864fc9e9646f0e0336 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 14:30:02 +1100 Subject: [PATCH 13/19] Fix aws command to tag AMIs --- .buildkite/steps/copy.sh | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index 22c18d6d2..ba6fbaeea 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -60,6 +60,18 @@ make_ami_public() { --launch-permission "{\"Add\": [{\"Group\":\"all\"}]}" } +tag-ami() { + local image_id="$1" + local region="$2" + local tag_key="$3" + local tag_value="$4" + + aws ec2 create-tags \ + --region "$region" \ + --resources "$image_id" \ + --tags "Key=$tag_key,Value=$tag_value" +} + if [[ -z "${BUILDKITE_AWS_STACK_BUCKET}" ]]; then echo "Must set an s3 bucket in BUILDKITE_AWS_STACK_BUCKET for temporary files" exit 1 @@ -118,19 +130,19 @@ EOF exit 0 fi -echo --- Tagging source images as released +echo --- Tagging AMIs as released if [[ $BUILDKITE_BRANCH == main || $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_IS_RELEASED:-false} == true ]]; then - aws ec2 tag-images --region "$source_region" --image-ids "$linux_amd64_source_image_id" --tags Key=IsReleased,Value=true - aws ec2 tag-images --region "$source_region" --image-ids "$linux_arm64_source_image_id" --tags Key=IsReleased,Value=true - aws ec2 tag-images --region "$source_region" --image-ids "$windows_amd64_source_image_id" --tags Key=IsReleased,Value=true + tag-ami "$linux_amd64_source_image_id" "$source_region" IsReleased true + tag-ami "$linux_arm64_source_image_id" "$source_region" IsReleased true + tag-ami "$windows_amd64_source_image_id" "$source_region" IsReleased true fi echo --- Tagging elastic ci stack release version echo Note: the same AMI may be used in multiple versions of the elastic stack, so we can\'t use the same tag key if [[ $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_VERSION:-false} == true ]]; then - aws ec2 tag-images --region "$source_region" --image-ids "$linux_amd64_source_image_id" --tags "Key=Version:${BUILDKITE_TAG},Value=true" - aws ec2 tag-images --region "$source_region" --image-ids "$linux_arm64_source_image_id" --tags "Key=Version:${BUILDKITE_TAG},Value=true" - aws ec2 tag-images --region "$source_region" --image-ids "$windows_amd64_source_image_id" --tags "Key=Version:${BUILDKITE_TAG},Value=true" + tag-ami "$linux_amd64_source_image_id" "$source_region" "Version:${BUILDKITE_TAG}" true + tag-ami "$linux_arm64_source_image_id" "$source_region" "Version:${BUILDKITE_TAG}" true + tag-ami "$windows_amd64_source_image_id" "$source_region" "Version:${BUILDKITE_TAG}" true fi echo --- Checking if there is a previously copy in the cache bucket From 21dc539860e56ecbe3eca20d02eb5df10d0ef164 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 15:23:41 +1100 Subject: [PATCH 14/19] Add a make recipe to print the agent version This is useful for debugging and other purposes. --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 2453b7425..bce017681 100644 --- a/Makefile +++ b/Makefile @@ -115,6 +115,10 @@ build/linux-arm64-ami.txt: packer-linux-arm64.output env-AWS_REGION CURRENT_AGENT_VERSION_LINUX ?= $(shell sed -En 's/^AGENT_VERSION="?(.+?)"?$$/\1/p' packer/linux/scripts/install-buildkite-agent.sh) CURRENT_AGENT_VERSION_WINDOWS ?= $(shell sed -En 's/^\$$AGENT_VERSION = "(.+?)"$$/\1/p' packer/windows/scripts/install-buildkite-agent.ps1) +print-agent-versions: + @echo Linux: $(CURRENT_AGENT_VERSION_LINUX) + @echo Windows: $(CURRENT_AGENT_VERSION_WINDOWS) + # Build linuxarm64 packer image packer-linux-arm64.output: $(PACKER_LINUX_FILES) docker run \ From 4efb2523245f372345e8178b3c09043b7d4195da Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 15:25:28 +1100 Subject: [PATCH 15/19] Print agent version at the start of packer builds --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index bce017681..d2317a78c 100644 --- a/Makefile +++ b/Makefile @@ -121,6 +121,7 @@ print-agent-versions: # Build linuxarm64 packer image packer-linux-arm64.output: $(PACKER_LINUX_FILES) + @echo Agent Version: $(CURRENT_AGENT_VERSION_LINUX) docker run \ -e AWS_DEFAULT_REGION \ -e AWS_PROFILE \ @@ -147,6 +148,7 @@ build/windows-amd64-ami.txt: packer-windows-amd64.output env-AWS_REGION # Build windows packer image packer-windows-amd64.output: $(PACKER_WINDOWS_FILES) + @echo Agent Version: $(CURRENT_AGENT_VERSION_WINDOWS) docker run \ -e AWS_DEFAULT_REGION \ -e AWS_PROFILE \ From 083d60e54f0f4f76de2b2e57d77aa90e792803fa Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 15:51:44 +1100 Subject: [PATCH 16/19] Formatting of a message --- .buildkite/steps/copy.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index ba6fbaeea..84b3d1ff8 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -138,7 +138,8 @@ if [[ $BUILDKITE_BRANCH == main || $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TA fi echo --- Tagging elastic ci stack release version -echo Note: the same AMI may be used in multiple versions of the elastic stack, so we can\'t use the same tag key +echo "Note: the same AMI may be used in multiple versions of the elastic stack," +echo "so we can't use the same tag key for each version." if [[ $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_VERSION:-false} == true ]]; then tag-ami "$linux_amd64_source_image_id" "$source_region" "Version:${BUILDKITE_TAG}" true tag-ami "$linux_arm64_source_image_id" "$source_region" "Version:${BUILDKITE_TAG}" true From 66a73fd7ebe8d1b87230bf40faf0e67c9843d98e Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 15:52:35 +1100 Subject: [PATCH 17/19] Remove an unnecessary use of escaped JSON --- .buildkite/steps/copy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index 84b3d1ff8..23e7c4595 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -57,7 +57,7 @@ make_ami_public() { aws ec2 modify-image-attribute \ --region "$region" \ --image-id "$image_id" \ - --launch-permission "{\"Add\": [{\"Group\":\"all\"}]}" + --launch-permission '{"Add": [{"Group": "all"}]}' } tag-ami() { From ffe4a6101b0cbd691d6275de1a7ebaf7055490d5 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 16:15:21 +1100 Subject: [PATCH 18/19] Quote arguments to echos While echo will print its arguments separated by spaces, if we need more than a single space, quotes are necessary. So for consistency, I will quote all arguments to echo from now on. --- .buildkite/steps/copy.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/steps/copy.sh b/.buildkite/steps/copy.sh index 23e7c4595..161581082 100755 --- a/.buildkite/steps/copy.sh +++ b/.buildkite/steps/copy.sh @@ -130,14 +130,14 @@ EOF exit 0 fi -echo --- Tagging AMIs as released +echo "--- Tagging AMIs as released" if [[ $BUILDKITE_BRANCH == main || $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_IS_RELEASED:-false} == true ]]; then tag-ami "$linux_amd64_source_image_id" "$source_region" IsReleased true tag-ami "$linux_arm64_source_image_id" "$source_region" IsReleased true tag-ami "$windows_amd64_source_image_id" "$source_region" IsReleased true fi -echo --- Tagging elastic ci stack release version +echo "--- Tagging elastic ci stack release version" echo "Note: the same AMI may be used in multiple versions of the elastic stack," echo "so we can't use the same tag key for each version." if [[ $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_VERSION:-false} == true ]]; then @@ -146,7 +146,7 @@ if [[ $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_VERSION:-false} == true ]]; tag-ami "$windows_amd64_source_image_id" "$source_region" "Version:${BUILDKITE_TAG}" true fi -echo --- Checking if there is a previously copy in the cache bucket +echo "--- Checking if there is a previously copy in the cache bucket" s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s-%s.yml" \ "${BUILDKITE_AWS_STACK_BUCKET}" \ "${linux_amd64_source_image_id}" \ @@ -159,7 +159,7 @@ if aws s3 cp "${s3_mappings_cache}" "$mapping_file"; then exit 0 fi -echo --- Copying images to other regions +echo "--- Copying images to other regions" # Get the image names to copy to other regions linux_amd64_source_image_name=$(get_image_name "$linux_amd64_source_image_id" "$source_region") From 819a22304be762c08eccf47ebe38caf7aca949be Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Thu, 19 Oct 2023 16:28:50 +1100 Subject: [PATCH 19/19] Fix sed option order --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d2317a78c..c81c0fef1 100644 --- a/Makefile +++ b/Makefile @@ -205,9 +205,9 @@ update-stack: build/aws-stack.yml env-STACK_NAME AGENT_VERSION ?= $(shell curl -Lfs "https://buildkite.com/agent/releases/latest?platform=linux&arch=amd64" | grep version | cut -d= -f2) bump-agent-version: - sed -iE "s/\[Buildkite Agent v.*\]/[Buildkite Agent v$(AGENT_VERSION)]/g" README.md - sed -iE "s/AGENT_VERSION=.+/AGENT_VERSION=$(AGENT_VERSION)/g" packer/linux/scripts/install-buildkite-agent.sh - sed -iE "s/\\\$$AGENT_VERSION = \".+\"/\$$AGENT_VERSION = \"$(AGENT_VERSION)\"/g" packer/windows/scripts/install-buildkite-agent.ps1 + sed -Ei "s/\[Buildkite Agent v.*\]/[Buildkite Agent v$(AGENT_VERSION)]/g" README.md + sed -Ei "s/AGENT_VERSION=.+/AGENT_VERSION=$(AGENT_VERSION)/g" packer/linux/scripts/install-buildkite-agent.sh + sed -Ei "s/\\\$$AGENT_VERSION = \".+\"/\$$AGENT_VERSION = \"$(AGENT_VERSION)\"/g" packer/windows/scripts/install-buildkite-agent.ps1 validate: build/aws-stack.yml aws --no-cli-pager cloudformation validate-template \