Merge pull request #1241 from buildkite/pdp-1837-copy-ami-tags-to-oth…

…er-regions-and-add-agent-version-as-a

.buildkite/pipeline.yaml

@@ -212,8 +212,8 @@ steps:
     artifact_paths: "build/*.yml"
     depends_on: "copy-ami"
 
-  - id: "cleanup"
-    name: "Cleanup"
+  - id: cleanup
+    name: ":broom: Cleanup"
     command: .buildkite/steps/cleanup.sh
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"

.buildkite/steps/copy.sh

@@ -11,11 +11,12 @@ copy_ami_to_region() {
   local destination_image_name="$4"
 
   aws ec2 copy-image \
+    --copy-image-tags \
     --source-image-id "$source_image_id" \
     --source-region "$source_region" \
     --name "$destination_image_name" \
     --region "$destination_image_region" \
-    --query "ImageId" \
+    --query ImageId \
     --output text
 }
 
@@ -56,7 +57,19 @@ make_ami_public() {
   aws ec2 modify-image-attribute \
     --region "$region" \
     --image-id "$image_id" \
-    --launch-permission "{\"Add\": [{\"Group\":\"all\"}]}"
+    --launch-permission '{"Add": [{"Group": "all"}]}'
+}
+
+tag-ami() {
+  local image_id="$1"
+  local region="$2"
+  local tag_key="$3"
+  local tag_value="$4"
+
+  aws ec2 create-tags \
+    --region "$region" \
+    --resources "$image_id" \
+    --tags "Key=$tag_key,Value=$tag_value"
 }
 
 if [[ -z "${BUILDKITE_AWS_STACK_BUCKET}" ]]; then
@@ -106,7 +119,7 @@ if [ $# -eq 0 ]; then
 fi
 
 # If we're not on the main branch or a tag build skip the copy
-if [[ $BUILDKITE_BRANCH != main ]] && [[ $BUILDKITE_TAG != "$BUILDKITE_BRANCH" ]] && [[ ${COPY_TO_ALL_REGIONS:-"false"} != "true" ]]; then
+if [[ $BUILDKITE_BRANCH != main && $BUILDKITE_TAG != "$BUILDKITE_BRANCH" && ${COPY_TO_ALL_REGIONS:-false} != true ]]; then
   echo "--- Skipping AMI copy on non-main/tag branch " >&2
   mkdir -p "$(dirname "$mapping_file")"
   cat <<EOF >"$mapping_file"
@@ -117,27 +130,44 @@ EOF
   exit 0
 fi
 
+echo "--- Tagging AMIs as released"
+if [[ $BUILDKITE_BRANCH == main || $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_IS_RELEASED:-false} == true ]]; then
+  tag-ami "$linux_amd64_source_image_id" "$source_region" IsReleased true
+  tag-ami "$linux_arm64_source_image_id" "$source_region" IsReleased true
+  tag-ami "$windows_amd64_source_image_id" "$source_region" IsReleased true
+fi
+
+echo "--- Tagging elastic ci stack release version"
+echo "Note: the same AMI may be used in multiple versions of the elastic stack,"
+echo "so we can't use the same tag key for each version."
+if [[ $BUILDKITE_TAG == "$BUILDKITE_BRANCH" || ${TAG_VERSION:-false} == true ]]; then
+  tag-ami "$linux_amd64_source_image_id" "$source_region" "Version:${BUILDKITE_TAG}" true
+  tag-ami "$linux_arm64_source_image_id" "$source_region" "Version:${BUILDKITE_TAG}" true
+  tag-ami "$windows_amd64_source_image_id" "$source_region" "Version:${BUILDKITE_TAG}" true
+fi
+
+echo "--- Checking if there is a previously copy in the cache bucket"
 s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s-%s.yml" \
   "${BUILDKITE_AWS_STACK_BUCKET}" \
   "${linux_amd64_source_image_id}" \
   "${linux_arm64_source_image_id}" \
   "${windows_amd64_source_image_id}" \
   "${BUILDKITE_BRANCH}")
 
-# Check if there is a previously copy in the cache bucket
 if aws s3 cp "${s3_mappings_cache}" "$mapping_file"; then
   echo "--- Skipping AMI copy, was previously copied"
   exit 0
 fi
 
+echo "--- Copying images to other regions"
+
 # Get the image names to copy to other regions
 linux_amd64_source_image_name=$(get_image_name "$linux_amd64_source_image_id" "$source_region")
 linux_arm64_source_image_name=$(get_image_name "$linux_arm64_source_image_id" "$source_region")
 windows_amd64_source_image_name=$(get_image_name "$windows_amd64_source_image_id" "$source_region")
 
 # Copy to all other regions
-# shellcheck disable=SC2048
-for region in ${ALL_REGIONS[*]}; do
+for region in "${ALL_REGIONS[@]}"; do
   if [[ $region != "$source_region" ]]; then
     echo "--- :linux: Copying Linux AMD64 $linux_amd64_source_image_id to $region" >&2
     IMAGES+=("$(copy_ami_to_region "$linux_amd64_source_image_id" "$source_region" "$region" "${linux_amd64_source_image_name}-${region}")")
@@ -160,8 +190,7 @@ Mappings:
 EOF
 
 echo "--- Waiting for AMIs to become available" >&2
-# shellcheck disable=SC2048
-for region in ${ALL_REGIONS[*]}; do
+for region in "${ALL_REGIONS[@]}"; do
   linux_amd64_image_id="${IMAGES[0]}"
   linux_arm64_image_id="${IMAGES[1]}"
   windows_amd64_image_id="${IMAGES[2]}"

Makefile

@@ -111,8 +111,17 @@ build/linux-arm64-ami.txt: packer-linux-arm64.output env-AWS_REGION
 	mkdir -p build
 	grep -Eo "$(AWS_REGION): (ami-.+)" $< | cut -d' ' -f2 | xargs echo -n > $@
 
+# NOTE: make removes the $ escapes, everything else is passed to the shell
+CURRENT_AGENT_VERSION_LINUX ?= $(shell sed -En 's/^AGENT_VERSION="?(.+?)"?$$/\1/p' packer/linux/scripts/install-buildkite-agent.sh)
+CURRENT_AGENT_VERSION_WINDOWS ?= $(shell sed -En 's/^\$$AGENT_VERSION = "(.+?)"$$/\1/p' packer/windows/scripts/install-buildkite-agent.ps1)
+
+print-agent-versions:
+	@echo Linux: $(CURRENT_AGENT_VERSION_LINUX)
+	@echo Windows: $(CURRENT_AGENT_VERSION_WINDOWS)
+
 # Build linuxarm64 packer image
 packer-linux-arm64.output: $(PACKER_LINUX_FILES)
+	@echo Agent Version: $(CURRENT_AGENT_VERSION_LINUX)
 	docker run \
 		-e AWS_DEFAULT_REGION  \
 		-e AWS_PROFILE \
@@ -130,6 +139,7 @@ packer-linux-arm64.output: $(PACKER_LINUX_FILES)
 			-var 'instance_type=$(ARM64_INSTANCE_TYPE)' \
 			-var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \
 			-var 'is_released=$(IS_RELEASED)' \
+			-var 'agent_version=$(CURRENT_AGENT_VERSION_LINUX)' \
 			buildkite-ami.pkr.hcl | tee $@
 
 build/windows-amd64-ami.txt: packer-windows-amd64.output env-AWS_REGION
@@ -138,6 +148,7 @@ build/windows-amd64-ami.txt: packer-windows-amd64.output env-AWS_REGION
 
 # Build windows packer image
 packer-windows-amd64.output: $(PACKER_WINDOWS_FILES)
+	@echo Agent Version: $(CURRENT_AGENT_VERSION_WINDOWS)
 	docker run \
 		-e AWS_DEFAULT_REGION  \
 		-e AWS_PROFILE \
@@ -155,6 +166,7 @@ packer-windows-amd64.output: $(PACKER_WINDOWS_FILES)
 			-var 'instance_type=$(WIN64_INSTANCE_TYPE)' \
 			-var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \
 			-var 'is_released=$(IS_RELEASED)' \
+			-var 'agent_version=$(CURRENT_AGENT_VERSION_WINDOWS)' \
 			buildkite-ami.pkr.hcl | tee $@
 
 # -----------------------------------------
@@ -193,12 +205,9 @@ update-stack: build/aws-stack.yml env-STACK_NAME
 AGENT_VERSION ?= $(shell curl -Lfs "https://buildkite.com/agent/releases/latest?platform=linux&arch=amd64" | grep version | cut -d= -f2)
 
 bump-agent-version:
-	sed -i.bak -E "s/\[Buildkite Agent v.*\]/[Buildkite Agent v$(AGENT_VERSION)]/g" README.md
-	sed -i.bak -E "s/AGENT_VERSION=.+/AGENT_VERSION=$(AGENT_VERSION)/g" packer/linux/scripts/install-buildkite-agent.sh
-	sed -i.bak -E "s/\\\$$AGENT_VERSION = \".+\"/\$$AGENT_VERSION = \"$(AGENT_VERSION)\"/g" packer/windows/scripts/install-buildkite-agent.ps1
-	rm README.md.bak packer/linux/scripts/install-buildkite-agent.sh.bak packer/windows/scripts/install-buildkite-agent.ps1.bak
-	git add README.md packer/linux/scripts/install-buildkite-agent.sh packer/windows/scripts/install-buildkite-agent.ps1
-	git commit -m "Bump buildkite-agent to v$(AGENT_VERSION)"
+	sed -Ei "s/\[Buildkite Agent v.*\]/[Buildkite Agent v$(AGENT_VERSION)]/g" README.md
+	sed -Ei "s/AGENT_VERSION=.+/AGENT_VERSION=$(AGENT_VERSION)/g" packer/linux/scripts/install-buildkite-agent.sh
+	sed -Ei "s/\\\$$AGENT_VERSION = \".+\"/\$$AGENT_VERSION = \"$(AGENT_VERSION)\"/g" packer/windows/scripts/install-buildkite-agent.ps1
 
 validate: build/aws-stack.yml
 	aws --no-cli-pager cloudformation validate-template \

packer/linux/buildkite-ami.pkr.hcl

@@ -27,6 +27,11 @@ variable "build_number" {
   default = "none"
 }
 
+variable "agent_version" {
+  type    = string
+  default = "devel"
+}
+
 variable "is_released" {
   type    = bool
   default = false
@@ -56,6 +61,7 @@ source "amazon-ebs" "elastic-ci-stack-ami" {
     Name          = "elastic-ci-stack-linux-${var.arch}"
     OSVersion     = "Amazon Linux 2023"
     BuildNumber   = var.build_number
+    AgentVersion  = var.agent_version
     IsReleased    = var.is_released
     SourceAMIID   = data.amazon-ami.al2023.id
     SourceAMIName = data.amazon-ami.al2023.name

packer/windows/buildkite-ami.pkr.hcl

@@ -27,6 +27,11 @@ variable "build_number" {
   default = "none"
 }
 
+variable "agent_version" {
+  type    = string
+  default = "devel"
+}
+
 variable "is_released" {
   type    = bool
   default = false
@@ -59,6 +64,7 @@ source "amazon-ebs" "elastic-ci-stack" {
     Name          = "elastic-ci-stack-windows"
     OSVersion     = "Windows Server 2019"
     BuildNumber   = var.build_number
+    AgentVersion  = var.agent_version
     IsReleased    = var.is_released
     SourceAMIID   = data.amazon-ami.windows-server-2019.id
     SourceAMIName = data.amazon-ami.windows-server-2019.name