Skip to content

Commit

Permalink
Enable remote tasks to be run in cluster
Browse files Browse the repository at this point in the history 
By default, we should run builds matching the local architecture
in-cluster to reduce the overhead of provisioning platforms. This will
enable a fully matrixed build for all images using only the remote
builds. This change will require the multi-platform controller to set
the /ssh/host to localhost in order for the builds to run in-cluster.

In a change from the prior behavior, we will now append a sanitized
version of the entire PLATFORM to the image tag upon request. We will no
longer try to extract just the arch from the PLATFORM as all platforms
may not follow the `os/arch` pattern. By appending the entire PLATFORM,
we remove any dependency on how local/remote platforms are configured. This
behavior is now needs to be explicitly requested.

Signed-off-by: arewm <arewm@users.noreply.github.com>
  • Loading branch information
@arewm
arewm committed Aug 15, 2024
1 parent 2d81397 commit 155e447
Show file tree
Hide file tree
Showing 5 changed files with 411 additions and 312 deletions.
115 changes: 66 additions & 49 deletions task-generator/remote/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,14 +102,16 @@ func convertToSsh(task *tektonapi.Task, taskVersion string) {
// The images produced in multi-platform builds need to have unique tags in order
// to prevent them from getting garbage collected before generating the image index.
// We can simplify this process, preventing the need for users to manually specify
// the image by auto-appending the architecture from the PLATFORM parameter. For
// example, this will append -arm64 if PLATFORM is linux/arm64 if not present. Since
// we cannot modify the parameter itself, this replacement needs to happen in any task
// step where the IMAGE parameter is used.
// If a user defines the IMAGE parameter with an -arm64 suffix, the arm64 suffix will
// not be appended again based on the PLATFORM.
adjustRemoteImage := `if [[ "${IMAGE##*-}" != "${PLATFORM##*/}" ]]; then
export IMAGE="${IMAGE}-${PLATFORM##*/}"
// the image by auto-appending the a sanitized PLATFORM parameter. For example, this
// will append linux-arm64 if PLATFORM is linux/arm64 and IMAGE_APPEND_PLATFORM is true.
// Many special characters are not allowed in tags so we will replace anything that
// isn't alphanumeric with a "-" to be safe. Since we cannot modify the parameter itself,
// this replacement needs to happen in any task step where the IMAGE parameter is used.
// IMAGE_APPEND_PLATFORM will be set to "false" by default so appending the platform is
// and explicit opt-in.
adjustRemoteImage := `if [ "${IMAGE_APPEND_PLATFORM}" == "true" ]; then
IMAGE="${IMAGE}-${PLATFORM//[^a-zA-Z0-9]/-}"
export IMAGE
fi
`

Expand Down Expand Up @@ -142,55 +144,65 @@ if [ -e "/ssh/error" ]; then
#no server could be provisioned
cat /ssh/error
exit 1
fi
export SSH_HOST=$(cat /ssh/host)
if [ "$SSH_HOST" == "localhost" ] ; then
IS_LOCALHOST=true
echo "Localhost detected; running build in cluster"
elif [ -e "/ssh/otp" ]; then
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
else
cp /ssh/id_rsa ~/.ssh
fi
chmod 0400 ~/.ssh/id_rsa
export SSH_HOST=$(cat /ssh/host)
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
mkdir -p scripts
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
if ! [[ $IS_LOCALHOST ]]; then
chmod 0400 ~/.ssh/id_rsa
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
`
if taskVersion != "0.1" {
ret += adjustRemoteImage
}
env := "$PODMAN_PORT_FORWARD \\\n"

// disable podman subscription-manager integration
env += " --tmpfs /run/secrets \\\n"
env += " --tmpfs /run/secrets \\\n"

// Before the build we sync the contents of the workspace to the remote host
for _, workspace := range task.Spec.Workspaces {
ret += "\nrsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
ret += "\n rsync -ra $(workspaces." + workspace.Name + ".path)/ \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/workspaces/" + workspace.Name + ":$(workspaces." + workspace.Name + ".path):Z\" \\\n"
}
// Also sync the volume mounts from the template
for _, volume := range task.Spec.StepTemplate.VolumeMounts {
ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
ret += "\n rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
}
for _, volume := range step.VolumeMounts {
if syncVolumes[volume.Name] {
ret += "\nrsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
ret += "\n rsync -ra " + volume.MountPath + "/ \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\""
podmanArgs += " -v \"$BUILD_DIR/volumes/" + volume.Name + ":" + volume.MountPath + ":Z\" \\\n"
}
}
ret += "\nrsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
podmanArgs += " -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
ret += "\nrsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/tekton-results/\""
podmanArgs += " -v \"$BUILD_DIR/tekton-results/:/tekton/results:Z\" \\\n"
ret += "\n rsync -ra \"$HOME/.docker/\" \"$SSH_HOST:$BUILD_DIR/.docker/\""
podmanArgs += " -v \"$BUILD_DIR/.docker/:/root/.docker:Z\" \\\n"
ret += "\n rsync -ra \"/tekton/results/\" \"$SSH_HOST:$BUILD_DIR/results/\""
podmanArgs += " -v \"$BUILD_DIR/results/:/tekton/results:Z\" \\\n"
ret += "\nfi\n"

if taskVersion != "0.1" {
ret += adjustRemoteImage
}

script := "scripts/script-" + step.Name + ".sh"

Expand All @@ -214,35 +226,37 @@ fi
ret += step.Script
ret += "\nbuildah push \"$IMAGE\" oci:rhtap-final-image"
ret += "\nREMOTESSHEOF"
ret += "\nchmod +x " + script
ret += "\nchmod +x " + script + "\n"

if task.Spec.StepTemplate != nil {
for _, e := range task.Spec.StepTemplate.Env {
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
}
}
ret += "\nrsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
containerScript := "/script/script-" + step.Name + ".sh"
ret += "\nif ! [[ $IS_LOCALHOST ]]; then"
ret += "\n rsync -ra scripts \"$SSH_HOST:$BUILD_DIR\""
containerScript := "scripts/script-" + step.Name + ".sh"
for _, e := range step.Env {
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
env += " -e " + e.Name + "=\"$" + e.Name + "\" \\\n"
}
podmanArgs += " -v $BUILD_DIR/scripts:/script:Z \\\n"
ret += "\nssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman run " + env + "" + podmanArgs + "--user=0 --rm \"$BUILDER_IMAGE\" " + containerScript
podmanArgs += " -v \"$BUILD_DIR/scripts:/scripts:Z\" \\\n"
ret += "\n ssh $SSH_ARGS \"$SSH_HOST\" $PORT_FORWARD podman run " + env + "" + podmanArgs + " --user=0 --rm \"$BUILDER_IMAGE\" /" + containerScript

// Sync the contents of the workspaces back so subsequent tasks can use them
for _, workspace := range task.Spec.Workspaces {
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/workspaces/" + workspace.Name + "/\" \"$(workspaces." + workspace.Name + ".path)/\""
}

for _, volume := range task.Spec.StepTemplate.VolumeMounts {
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/volumes/" + volume.Name + "/\" " + volume.MountPath + "/"
}
//sync back results
ret += "\nrsync -ra \"$SSH_HOST:$BUILD_DIR/tekton-results/\" \"/tekton/results/\""
ret += "\n rsync -ra \"$SSH_HOST:$BUILD_DIR/results/\" \"/tekton/results/\""

ret += "\nbuildah pull oci:rhtap-final-image"
ret += "\n buildah pull oci:rhtap-final-image"
ret += "\nelse\n bash " + containerScript
ret += "\nfi"
ret += "\nbuildah images"
ret += "\nbuildah tag localhost/rhtap-final-image \"$IMAGE\""
ret += "\ncontainer=$(buildah from --pull-never \"$IMAGE\")\nbuildah mount \"$container\" | tee /shared/container_path\necho $container > /shared/container_name"

for _, i := range strings.Split(ret, "\n") {
Expand Down Expand Up @@ -276,5 +290,8 @@ fi
task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "BUILDER_IMAGE", Value: builderImage})
if taskVersion != "0.1" {
task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "PLATFORM", Value: "$(params.PLATFORM)"})

task.Spec.Params = append(task.Spec.Params, tektonapi.ParamSpec{Name: "IMAGE_APPEND_PLATFORM", Type: tektonapi.ParamTypeString, Description: "Whether to append a sanitized platform architecture on the IMAGE tag", Default: &tektonapi.ParamValue{StringVal: "false", Type: tektonapi.ParamTypeString}})
task.Spec.StepTemplate.Env = append(task.Spec.StepTemplate.Env, v1.EnvVar{Name: "IMAGE_APPEND_PLATFORM", Value: "$(params.IMAGE_APPEND_PLATFORM)"})
}
}
132 changes: 73 additions & 59 deletions task/buildah-remote-oci-ta/0.1/buildah-remote-oci-ta.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,35 +220,45 @@ spec:
#no server could be provisioned
cat /ssh/error
exit 1
fi
export SSH_HOST=$(cat /ssh/host)
if [ "$SSH_HOST" == "localhost" ] ; then
IS_LOCALHOST=true
echo "Localhost detected; running build in cluster"
elif [ -e "/ssh/otp" ]; then
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
curl --cacert /ssh/otp-ca -XPOST -d @/ssh/otp $(cat /ssh/otp-server) >~/.ssh/id_rsa
echo "" >> ~/.ssh/id_rsa
else
cp /ssh/id_rsa ~/.ssh
fi
chmod 0400 ~/.ssh/id_rsa
export SSH_HOST=$(cat /ssh/host)
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
mkdir -p scripts
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
if ! [[ $IS_LOCALHOST ]]; then
chmod 0400 ~/.ssh/id_rsa
export BUILD_DIR=$(cat /ssh/user-dir)
export SSH_ARGS="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=10"
echo "$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" mkdir -p "$BUILD_DIR/workspaces" "$BUILD_DIR/scripts" "$BUILD_DIR/volumes"
PORT_FORWARD=""
PODMAN_PORT_FORWARD=""
if [ -n "$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR" ] ; then
PORT_FORWARD=" -L 80:$JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR:80"
PODMAN_PORT_FORWARD=" -e JVM_BUILD_WORKSPACE_ARTIFACT_CACHE_PORT_80_TCP_ADDR=localhost"
fi
rsync -ra /shared/ "$SSH_HOST:$BUILD_DIR/volumes/shared/"
rsync -ra /var/workdir/ "$SSH_HOST:$BUILD_DIR/volumes/workdir/"
rsync -ra /entitlement/ "$SSH_HOST:$BUILD_DIR/volumes/etc-pki-entitlement/"
rsync -ra /activation-key/ "$SSH_HOST:$BUILD_DIR/volumes/activation-key/"
rsync -ra /additional-secret/ "$SSH_HOST:$BUILD_DIR/volumes/additional-secret/"
rsync -ra /mnt/trusted-ca/ "$SSH_HOST:$BUILD_DIR/volumes/trusted-ca/"
rsync -ra "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/"
rsync -ra "/tekton/results/" "$SSH_HOST:$BUILD_DIR/results/"
fi
rsync -ra /shared/ "$SSH_HOST:$BUILD_DIR/volumes/shared/"
rsync -ra /var/workdir/ "$SSH_HOST:$BUILD_DIR/volumes/workdir/"
rsync -ra /entitlement/ "$SSH_HOST:$BUILD_DIR/volumes/etc-pki-entitlement/"
rsync -ra /activation-key/ "$SSH_HOST:$BUILD_DIR/volumes/activation-key/"
rsync -ra /additional-secret/ "$SSH_HOST:$BUILD_DIR/volumes/additional-secret/"
rsync -ra /mnt/trusted-ca/ "$SSH_HOST:$BUILD_DIR/volumes/trusted-ca/"
rsync -ra "$HOME/.docker/" "$SSH_HOST:$BUILD_DIR/.docker/"
rsync -ra "/tekton/results/" "$SSH_HOST:$BUILD_DIR/tekton-results/"
cat >scripts/script-build.sh <<'REMOTESSHEOF'
#!/bin/bash
set -o verbose
Expand Down Expand Up @@ -431,44 +441,48 @@ spec:
buildah push "$IMAGE" oci:rhtap-final-image
REMOTESSHEOF
chmod +x scripts/script-build.sh
rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman run $PODMAN_PORT_FORWARD \
--tmpfs /run/secrets \
-e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
-e ADD_CAPABILITIES="$ADD_CAPABILITIES" \
-e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
-e BUILD_ARGS_FILE="$BUILD_ARGS_FILE" \
-e CONTEXT="$CONTEXT" \
-e DOCKERFILE="$DOCKERFILE" \
-e ENTITLEMENT_SECRET="$ENTITLEMENT_SECRET" \
-e HERMETIC="$HERMETIC" \
-e IMAGE="$IMAGE" \
-e IMAGE_EXPIRES_AFTER="$IMAGE_EXPIRES_AFTER" \
-e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
-e SQUASH="$SQUASH" \
-e STORAGE_DRIVER="$STORAGE_DRIVER" \
-e TARGET_STAGE="$TARGET_STAGE" \
-e TLSVERIFY="$TLSVERIFY" \
-e YUM_REPOS_D_FETCHED="$YUM_REPOS_D_FETCHED" \
-e YUM_REPOS_D_SRC="$YUM_REPOS_D_SRC" \
-e YUM_REPOS_D_TARGET="$YUM_REPOS_D_TARGET" \
-e COMMIT_SHA="$COMMIT_SHA" \
-v "$BUILD_DIR/volumes/shared:/shared:Z" \
-v "$BUILD_DIR/volumes/workdir:/var/workdir:Z" \
-v "$BUILD_DIR/volumes/etc-pki-entitlement:/entitlement:Z" \
-v "$BUILD_DIR/volumes/activation-key:/activation-key:Z" \
-v "$BUILD_DIR/volumes/additional-secret:/additional-secret:Z" \
-v "$BUILD_DIR/volumes/trusted-ca:/mnt/trusted-ca:Z" \
-v "$BUILD_DIR/.docker/:/root/.docker:Z" \
-v "$BUILD_DIR/tekton-results/:/tekton/results:Z" \
-v $BUILD_DIR/scripts:/script:Z \
--user=0 --rm "$BUILDER_IMAGE" /script/script-build.sh
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/
rsync -ra "$SSH_HOST:$BUILD_DIR/tekton-results/" "/tekton/results/"
buildah pull oci:rhtap-final-image
if ! [[ $IS_LOCALHOST ]]; then
rsync -ra scripts "$SSH_HOST:$BUILD_DIR"
ssh $SSH_ARGS "$SSH_HOST" $PORT_FORWARD podman run $PODMAN_PORT_FORWARD \
--tmpfs /run/secrets \
-e ADDITIONAL_SECRET="$ADDITIONAL_SECRET" \
-e ADD_CAPABILITIES="$ADD_CAPABILITIES" \
-e BUILDAH_FORMAT="$BUILDAH_FORMAT" \
-e BUILD_ARGS_FILE="$BUILD_ARGS_FILE" \
-e CONTEXT="$CONTEXT" \
-e DOCKERFILE="$DOCKERFILE" \
-e ENTITLEMENT_SECRET="$ENTITLEMENT_SECRET" \
-e HERMETIC="$HERMETIC" \
-e IMAGE="$IMAGE" \
-e IMAGE_EXPIRES_AFTER="$IMAGE_EXPIRES_AFTER" \
-e SKIP_UNUSED_STAGES="$SKIP_UNUSED_STAGES" \
-e SQUASH="$SQUASH" \
-e STORAGE_DRIVER="$STORAGE_DRIVER" \
-e TARGET_STAGE="$TARGET_STAGE" \
-e TLSVERIFY="$TLSVERIFY" \
-e YUM_REPOS_D_FETCHED="$YUM_REPOS_D_FETCHED" \
-e YUM_REPOS_D_SRC="$YUM_REPOS_D_SRC" \
-e YUM_REPOS_D_TARGET="$YUM_REPOS_D_TARGET" \
-e COMMIT_SHA="$COMMIT_SHA" \
-v "$BUILD_DIR/volumes/shared:/shared:Z" \
-v "$BUILD_DIR/volumes/workdir:/var/workdir:Z" \
-v "$BUILD_DIR/volumes/etc-pki-entitlement:/entitlement:Z" \
-v "$BUILD_DIR/volumes/activation-key:/activation-key:Z" \
-v "$BUILD_DIR/volumes/additional-secret:/additional-secret:Z" \
-v "$BUILD_DIR/volumes/trusted-ca:/mnt/trusted-ca:Z" \
-v "$BUILD_DIR/.docker/:/root/.docker:Z" \
-v "$BUILD_DIR/results/:/tekton/results:Z" \
-v "$BUILD_DIR/scripts:/scripts:Z" \
--user=0 --rm "$BUILDER_IMAGE" /scripts/script-build.sh
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/shared/" /shared/
rsync -ra "$SSH_HOST:$BUILD_DIR/volumes/workdir/" /var/workdir/
rsync -ra "$SSH_HOST:$BUILD_DIR/results/" "/tekton/results/"
buildah pull oci:rhtap-final-image
else
bash scripts/script-build.sh
fi
buildah images
buildah tag localhost/rhtap-final-image "$IMAGE"
container=$(buildah from --pull-never "$IMAGE")
buildah mount "$container" | tee /shared/container_path
echo $container > /shared/container_name
Expand Down
Loading

0 comments on commit 155e447

Please sign in to comment.