diff --git a/.dockerignore b/.dockerignore index 7463dc83..ea070911 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,8 +8,3 @@ examples/ helpers/ scripts/ test/ -go.mod -go.sum -Dockerfile -Makefile -VERSION diff --git a/.github/workflows/csi-integration-tests.yml b/.github/workflows/csi-integration-tests.yml new file mode 100644 index 00000000..7e3813ed --- /dev/null +++ b/.github/workflows/csi-integration-tests.yml @@ -0,0 +1,175 @@ +name: CSI Integration Tests + +on: + push: + branches: + - master + + pull_request: + + # Allow to run this workflow manually from the Actions tab + workflow_dispatch: + + # Run this regularly, to get integration tests results against new + # Kubernetes releases. + schedule: + - cron: '15 3 * * *' + +permissions: + contents: read + +jobs: + test-matrix: + name: "Get Kubernetes Releases" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: "Generate Test Matrix" + id: list + run: 'echo "tests=$(helpers/test-matrix)" >> $GITHUB_OUTPUT' + + outputs: + tests: ${{ steps.list.outputs.tests }} + + build-image: + name: "Build Container Image" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Evaluate image name + run: 'helpers/image-from-ref >> $GITHUB_ENV' + + - name: Extract version + run: echo "VERSION=$(cat VERSION)" >> $GITHUB_ENV + + - name: Build image + run: make build + + - name: Export image + run: 'docker image save "$IMAGE" -o image.tar' + + - name: Store hash + run: 'shasum -a 256 image.tar | tee image.tar.sha256' + + - name: Store image + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: tested-image + path: | + image.tar + image.tar.sha256 + retention-days: 30d + + check-csi-integration: + # Preflight: verify the CLOUDSCALE_API_TOKEN at step-level and set an output + # so the integration job can be skipped. + # + # GitHub Actions limitations motivating this: + # - `secrets` are NOT available in `jobs..if` (job-level `if`), so you cannot + # directly gate/skip a job by testing a secret there. + name: Check CSI Integration Configuration + runs-on: ubuntu-latest + + outputs: + integration-enabled: ${{ steps.check.outputs.enabled }} + + steps: + - id: check + name: Verify CLOUDSCALE_API_TOKEN is present + env: + CLOUDSCALE_API_TOKEN: ${{ secrets.CLOUDSCALE_API_TOKEN }} + run: | + if [ -n "$CLOUDSCALE_API_TOKEN" ]; then + echo "enabled=true" >> $GITHUB_OUTPUT + echo "CLOUDSCALE_API_TOKEN found — integration will run." + else + echo "enabled=false" >> $GITHUB_OUTPUT + echo "CLOUDSCALE_API_TOKEN not configured — skipping integration." + fi + + integration: + name: "Kubernetes ${{ matrix.kubernetes }}" + runs-on: ubuntu-latest + + needs: + - test-matrix + - build-image + - check-csi-integration + if: needs.check-csi-integration.outputs.integration-enabled == 'true' + + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: "${{ fromJson(needs.test-matrix.outputs.tests) }}" + + env: + CLOUDSCALE_API_TOKEN: ${{ secrets.CLOUDSCALE_API_TOKEN }} + KUBERNETES: '${{ matrix.kubernetes }}' + SUBNET: '${{ matrix.subnet }}' + CLUSTER_PREFIX: '${{ matrix.cluster_prefix }}' + IMAGE_SOURCE: import + + # Prevent integration tests from running in parallel. Ideally this should + # be seuqential, but that won't work due to the following issue: + # + # https://github.com/orgs/community/discussions/5435 + # + # Instead we ensure that only one integration test per supported version + # is run at any given time. + concurrency: + group: integration-${{ matrix.kubernetes }} + + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Load image + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + name: tested-image + + - name: Validate hash + run: 'shasum --check image.tar.sha256' + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + + - name: Setup Helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 + with: + version: v4.2.4 + + - name: Evaluate image name + run: 'helpers/image-from-ref >> $GITHUB_ENV' + + - name: Cleanup Leftovers + if: always() + run: helpers/cleanup + + - name: Create Test Cluster + run: helpers/run-in-test-cluster + + - name: Wait For CCM/CSI Startup + run: sleep 60 + + - name: Run Integration Tests + run: make test-integration + + - name: Wait For Kubernetes-Internal Cleanup + if: always() + run: sleep 30 + + - name: Destroy Test Cluster + if: always() + run: helpers/cleanup diff --git a/.gitignore b/.gitignore index ad951bc7..d7af6dbb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ charts/csi-cloudscale/charts cmd/cloudscale-csi-plugin/cloudscale-csi-plugin k8test/ bin/ + +__pycache__/ diff --git a/Makefile b/Makefile index 8797c8e5..73cf710b 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,7 @@ NAME=cloudscale-csi-plugin OS ?= linux GO_VERSION := $(shell awk '/^go/ {print $$2}' go.mod) -ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),) - GIT_TREE_STATE=clean -else - GIT_TREE_STATE=dirty -endif +GIT_TREE_STATE ?= $(shell git status --porcelain 2>/dev/null | grep -q . && echo "dirty" || echo "clean") COMMIT ?= $(shell git rev-parse HEAD) BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) LDFLAGS ?= -X github.com/cloudscale-ch/csi-cloudscale/driver.version=${VERSION} -X github.com/cloudscale-ch/csi-cloudscale/driver.commit=${COMMIT} -X github.com/cloudscale-ch/csi-cloudscale/driver.gitTreeState=${GIT_TREE_STATE} @@ -14,6 +10,7 @@ PKG ?= github.com/cloudscale-ch/csi-cloudscale/cmd/cloudscale-csi-plugin VERSION ?= $(shell cat VERSION) CHART_VERSION ?= $(shell awk '/^version:/ {print $$2}' charts/csi-cloudscale/Chart.yaml) DOCKER_REPO ?= quay.io/cloudscalech/cloudscale-csi-plugin +IMAGE ?= $(DOCKER_REPO):$(VERSION) # Setting SHELL to bash allows bash commands to be executed by recipes. # Options are set to exit when a recipe line exits non-zero or a piped command fails. @@ -115,13 +112,18 @@ govulncheck: govulncheck-tool ## Run govulncheck (advisory only) .PHONY: compile compile: ## Build the project binary - @echo "==> Building the project" - @docker run --rm -it -e GOOS=${OS} -e GOARCH=amd64 -v ${PWD}/:/app -w /app golang:${GO_VERSION}-alpine sh -c 'apk add git && go build -o cmd/cloudscale-csi-plugin/${NAME} -ldflags "$(LDFLAGS)" ${PKG}' + @echo "==> Building the binary" + CGO_ENABLED=0 go build -o cmd/cloudscale-csi-plugin/$(NAME) -ldflags "$(LDFLAGS)" $(PKG) .PHONY: build -build: compile ## Build the docker image +build: ## Build the docker image @echo "==> Building the docker image" - @docker build --platform linux/amd64 -t $(DOCKER_REPO):$(VERSION) cmd/cloudscale-csi-plugin -f cmd/cloudscale-csi-plugin/Dockerfile + docker build --platform linux/amd64 \ + -t $(IMAGE) \ + --build-arg VERSION="$(VERSION)" \ + --build-arg COMMIT="$(COMMIT)" \ + --build-arg GIT_TREE_STATE="$(GIT_TREE_STATE)" \ + -f cmd/cloudscale-csi-plugin/Dockerfile . .PHONY: push push: ## Push docker image to registry @@ -132,9 +134,9 @@ ifeq ($(DOCKER_REPO),quay.io/cloudscalech/cloudscale-csi-plugin) endif endif endif - @echo "==> Publishing $(DOCKER_REPO):$(VERSION)" - @docker push $(DOCKER_REPO):$(VERSION) - @echo "==> Your image is now available at $(DOCKER_REPO):$(VERSION)" + @echo "==> Publishing $(IMAGE)" + @docker push $(IMAGE) + @echo "==> Your image is now available at $(IMAGE)" .PHONY: publish publish: build push clean ## Build, push, and clean diff --git a/README.md b/README.md index 3d85e338..9b0c7d10 100644 --- a/README.md +++ b/README.md @@ -418,11 +418,31 @@ This will create a binary with version `dev` and docker image pushed to `cloudscalech/cloudscale-csi-plugin:dev` -To run the integration tests run the following: +To run the integration tests locally with your local CSI build: -``` -$ export KUBECONFIG=$(pwd)/kubeconfig +```bash +# 1. Build and export the CSI image +$ VERSION=dev make publish + +# 2. Create a test cluster with CCM and CSI +$ export CLOUDSCALE_API_TOKEN=your-token +$ export IMAGE=quay.io/cloudscalech/cloudscale-csi-plugin:dev +$ helpers/run-in-test-cluster + +# This will: +# - Create a Kubernetes cluster on cloudscale.ch +# - Deploy CCM from the latest official release +# - Deploy CSI from your local build + +# 3. Run integration tests +$ export KUBECONFIG=$(pwd)/k8test/cluster/admin.conf +$ make test-integration + +# Run a single test $ TESTARGS='-run TestPod_Single_SSD_Volume' make test-integration + +# 4. Clean up +$ helpers/cleanup ``` diff --git a/cmd/cloudscale-csi-plugin/Dockerfile b/cmd/cloudscale-csi-plugin/Dockerfile index 9d173048..9c2bd43a 100644 --- a/cmd/cloudscale-csi-plugin/Dockerfile +++ b/cmd/cloudscale-csi-plugin/Dockerfile @@ -1,30 +1,41 @@ -# Copyright 2018 DigitalOcean -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +FROM golang:1.27-alpine AS builder + +RUN apk add --no-cache git make bash + +WORKDIR /src + +# Copy go.mod/go.sum first for better layer caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy all source code +COPY . . + +# Build arguments for version information +ARG VERSION=dev +ARG COMMIT=unknown +ARG GIT_TREE_STATE=unknown + +# Build using make (ensures consistent build logic) +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make compile \ + VERSION="${VERSION}" \ + COMMIT="${COMMIT}" \ + GIT_TREE_STATE="${GIT_TREE_STATE}" + FROM alpine:3.23.3 # e2fsprogs-extra is required for resize2fs used for the resize operation # blkid: block device identification tool from util-linux RUN apk add --no-cache ca-certificates \ - e2fsprogs \ - findmnt \ - xfsprogs \ - cryptsetup \ - udev \ - blkid \ - e2fsprogs-extra - -ADD cloudscale-csi-plugin /bin/ -ADD csi-diskinfo.sh /bin/ + e2fsprogs \ + findmnt \ + xfsprogs \ + cryptsetup \ + udev \ + blkid \ + e2fsprogs-extra + +COPY --from=builder /src/cmd/cloudscale-csi-plugin/cloudscale-csi-plugin /bin/ +COPY --from=builder /src/cmd/cloudscale-csi-plugin/csi-diskinfo.sh /bin/ ENTRYPOINT ["/bin/cloudscale-csi-plugin"] diff --git a/helpers/bootstrap-cluster b/helpers/bootstrap-cluster deleted file mode 100755 index 5f9a4879..00000000 --- a/helpers/bootstrap-cluster +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash -# -# Ensures that a Kubernetes test cluster is present and updated -# -set -euo pipefail - -# Default values -RANDOM_NUMBER=$((RANDOM % 8193)) -K8TEST_SHA="0bf850b" -ZONE="lpg1" -CLUSTER_PREFIX="csi-test-$RANDOM_NUMBER" -KUBERNETES="latest" -FLAVOR="plus-8-4" -CONTROL_COUNT=1 -WORKER_COUNT=3 -IMAGE="ubuntu-22.04" -VOLUME_SIZE_GB="25" - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - --k8test-sha) - K8TEST_SHA="$2" - shift - shift - ;; - --zone) - ZONE="$2" - shift - shift - ;; - --cluster-prefix) - CLUSTER_PREFIX="$2" - shift - shift - ;; - --kubernetes) - KUBERNETES="$2" - shift - shift - ;; - --control-count) - CONTROL_COUNT="$2" - shift - shift - ;; - --worker-count) - WORKER_COUNT="$2" - shift - shift - ;; - --image) - IMAGE="$2" - shift - shift - ;; - --flavor) - FLAVOR="$2" - shift - shift - ;; - --volume-size-gb) - VOLUME_SIZE_GB="$2" - shift - shift - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac -done - -# Prepares k8test with an existing virtual env, or a newly created on -function ensure-k8test() { - if ! test -d k8test; then - git clone git@github.com:cloudscale-ch/k8test.git - git -C k8test checkout "$K8TEST_SHA" - fi - - if [[ "${VIRTUAL_ENV:-}" == "" ]]; then - - if ! test -d k8test/venv; then - python3.12 -m venv k8test/venv - fi - - # shellcheck source=/dev/null - source k8test/venv/bin/activate - fi - - if ! command -v poetry > /dev/null; then - python3.12 -m pip install poetry - fi - - if ! command -v ansible > /dev/null; then - poetry install --directory k8test - fi -} - -# Launches the test cluster, if there's no inventory yet -function ensure-inventory() { - if ! test -d k8test/cluster; then - mkdir k8test/cluster - fi - - if ! test -f k8test/cluster/ssh.pub; then - ssh-keygen -t ed25519 -N '' -f k8test/cluster/ssh - fi - - if ! test -f k8test/cluster/inventory.yml; then - k8test/playbooks/create-cluster.yml \ - -e ssh_key=k8test/cluster/ssh.pub \ - -e zone="$ZONE" \ - -e cluster_prefix="$CLUSTER_PREFIX" \ - -e kubernetes="$KUBERNETES" \ - -e control_count="$CONTROL_COUNT" \ - -e worker_count="$WORKER_COUNT" \ - -e image="$IMAGE" \ - -e flavor="$FLAVOR" \ - -e volume_size_gb="$VOLUME_SIZE_GB" - - # Those won't really change between runs, so update them during install - k8test/playbooks/update-secrets.yml \ - -i k8test/cluster/inventory.yml - fi -} - -ensure-k8test -ensure-inventory diff --git a/helpers/clean-up b/helpers/clean-up deleted file mode 100755 index 5605e086..00000000 --- a/helpers/clean-up +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# -# Ensures that any Kubernetes cluster is cleaned up -# -set -euo pipefail - -if test -f k8test/cluster/inventory.yml; then - source k8test/venv/bin/activate - k8test/playbooks/destroy-cluster.yml -i k8test/cluster/inventory.yml -fi - -if test -d k8test; then - rm -rf k8test -fi diff --git a/helpers/cleanup b/helpers/cleanup new file mode 100755 index 00000000..94c1afe3 --- /dev/null +++ b/helpers/cleanup @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# Ensures that any Kubernetes cluster is cleaned up +# +set -euo pipefail + +export CLUSTER_PREFIX="${CLUSTER_PREFIX-k8test}" + +# Make sure the k8test/helpers/release-set CLI is available +source helpers/run-in-test-cluster +ensure-k8test > /dev/null + +for attempt in 1 2 3; do + echo "Destroy attempt ${attempt}..." + + if k8test/playbooks/destroy-cluster.yml \ + -i k8test/cluster/inventory.yml \ + -e cluster_prefix="$CLUSTER_PREFIX"; then + exit 0 + fi + + echo "Cleanup failed, retrying in 60s..." + sleep 60 +done + +echo "Cleanup failed after 3 attempts." +exit 1 diff --git a/helpers/image-from-ref b/helpers/image-from-ref new file mode 100755 index 00000000..b3fbaac2 --- /dev/null +++ b/helpers/image-from-ref @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +""" Looks at the GITHUB_REF environment variable and determins the image name +that should be used for the pipeline. + +For more information: +https://docs.github.com/en/actions/learn-github-actions/contexts#github-context + +The output is meant to be used with $GITHUB_ENV. + +""" + +import argparse +import os +import sys + +REPOSITORY = 'quay.io/cloudscalech/cloudscale-csi-plugin' + +parser = argparse.ArgumentParser(usage=__doc__) +parser.add_argument('--ref', default=os.environ.get('GITHUB_REF')) + + +def main(ref: str) -> int: + if not ref: + print("Either use --ref or set GITHUB_REF") + return 1 + + match ref.split('/'): + case ["refs", "heads", *branch_parts]: + tag = f"branch-{'-'.join(branch_parts)}" + case ["refs", "pull", pull_request, "merge"]: + tag = f"pull-request-{pull_request}" + case ["refs", "tags", tag_name]: + tag = tag_name + case _: + print(f"'{ref}' did not match any known pattern") + return 1 + + print(f"IMAGE={REPOSITORY}:{tag}") + return 0 + + +if __name__ == '__main__': + sys.exit(main(**vars(parser.parse_args()))) diff --git a/helpers/run-in-test-cluster b/helpers/run-in-test-cluster new file mode 100755 index 00000000..44eaee01 --- /dev/null +++ b/helpers/run-in-test-cluster @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +# +# Ensures that a Kubernetes test cluster is present and updated +# +# shellcheck source=/dev/null +set -euo pipefail + +export ANSIBLE_CONFIG="$PWD"/k8test/ansible.cfg +export KUBERNETES="${KUBERNETES-latest}" +export CLUSTER_PREFIX="${CLUSTER_PREFIX-csi}" +export SUBNET="${SUBNET-10.200.1.0/24}" +export CSI_IMAGE="${IMAGE-quay.io/cloudscalech/cloudscale-csi-plugin:test}" +export IMAGE="${IMAGE-quay.io/cloudscalech/cloudscale-csi-plugin:test}" +export CCM_RELEASE_URL="https://github.com/cloudscale-ch/cloudscale-cloud-controller-manager/releases/latest/download/config.yml" + +# Prepares k8test with an existing virtual env, or a newly created on +function ensure-k8test() { + + # On GitHub actions, things are a bit different. Ansible is actually found + # on the runners and cloning via SSH does not work. + # + # Therefore we run a separate install block that is only meant for GitHub. + if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + if test -d k8test; then + source k8test/venv/bin/activate + return + fi + + mkdir -p ~/.ssh/ && touch ~/.ssh/known_hosts + git clone https://github.com/cloudscale-ch/k8test + python3 -m venv k8test/venv + source k8test/venv/bin/activate + pip install poetry + poetry install --directory k8test + return + fi + + if ! test -d k8test; then + git clone git@github.com:cloudscale-ch/k8test.git + fi + + if [[ "${VIRTUAL_ENV:-}" == "" ]]; then + + if ! test -d k8test/venv; then + python3 -m venv k8test/venv + fi + + source k8test/venv/bin/activate + fi + + if ! command -v poetry > /dev/null; then + pip install poetry + fi + + if ! command -v ansible > /dev/null; then + poetry install --directory k8test + fi +} + +# Prints a random zone +function random-zone() { + arr[0]="rma1" + arr[1]="lpg1" + + rand=$((RANDOM % 2)) + echo "${arr[$rand]}" +} + +# Launches the test cluster, if there's no admin.conf yet +function ensure-cluster() { + if ! test -d k8test/cluster; then + mkdir k8test/cluster + fi + + if ! test -f k8test/cluster/ssh.pub; then + ssh-keygen -t ed25519 -N '' -f k8test/cluster/ssh + fi + + if ! test -f k8test/cluster/admin.conf; then + zone="$(random-zone)" + + # First create the cluster, without configuring the network + k8test/playbooks/create-cluster.yml \ + -e zone="$zone" \ + -e ssh_key=k8test/cluster/ssh.pub \ + -e control_count=2 \ + -e worker_count=2 \ + -e kubelet_extra_args='--cloud-provider=external' \ + -e kubernetes="${KUBERNETES}" \ + -e cluster_prefix="${CLUSTER_PREFIX}" \ + -e subnet="${SUBNET}" \ + --tags setup-vms,setup-controls + + # Those won't really change between runs, so update them during install + k8test/playbooks/update-secrets.yml \ + -i k8test/cluster/inventory.yml + + # Second, get the CCM running. Without it setting the node IPs, Cilium + # won't work. + deploy-ccm + + # Finally, setup the cluster to completion (includes Cilium install) + k8test/playbooks/create-cluster.yml \ + -e zone="$zone" \ + -e ssh_key=k8test/cluster/ssh.pub \ + -e control_count=2 \ + -e worker_count=2 \ + -e kubelet_extra_args='--cloud-provider=external' \ + -e kubernetes="${KUBERNETES}" \ + -e cluster_prefix="${CLUSTER_PREFIX}" \ + -e subnet="${SUBNET}" + fi +} + +# Build the latest image each time +function build-image() { + k8test/playbooks/build-image.yml \ + -i k8test/cluster/inventory.yml \ + -e dockerfile=./Dockerfile \ + -e tag="$IMAGE" \ + -e extra='--build-arg=VERSION=test' \ + -l controls +} + +# Import an image from the host +function import-image() { + k8test/playbooks/import-image.yml \ + -i k8test/cluster/inventory.yml \ + -e image="$PWD/image.tar" +} + +# Deploy CCM from official GitHub release +function deploy-ccm() { + export KUBECONFIG=k8test/cluster/admin.conf + + echo "Deploying CCM from ${CCM_RELEASE_URL}..." + kubectl apply -f "${CCM_RELEASE_URL}" + + echo "Waiting for CCM to be ready..." + kubectl -n kube-system rollout status daemonset/cloudscale-cloud-controller-manager --timeout=120s +} + +# Deploy VolumeSnapshot CRDs and snapshot controller +function deploy-snapshot-crds() { + export KUBECONFIG=k8test/cluster/admin.conf + + echo "Installing VolumeSnapshot CRDs..." + kubectl apply -k "https://github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v8.4.0" + + echo "Installing snapshot controller..." + kubectl apply -k "https://github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v8.4.0" +} + +# Deploy CSI using Helm chart with custom image +function deploy-csi() { + api_url=$(echo "${CLOUDSCALE_API_URL-https://api.cloudscale.ch}" | sed 's|/v1||g') + + export KUBECONFIG=k8test/cluster/admin.conf + + # Parse CSI_IMAGE to extract tag + local image_tag="${CSI_IMAGE##*:}" + + echo "Deploying CSI with image tag: ${image_tag}..." + + # Generate manifest from Helm chart with custom image + helm template csi-cloudscale -n kube-system \ + --set nameOverride=csi-cloudscale \ + --set controller.image.tag="${image_tag}" \ + --set node.image.tag="${image_tag}" \ + --set cloudscale.apiUrl="${api_url}/" \ + ./charts/csi-cloudscale | kubectl apply -n kube-system -f - + + echo "Waiting for CSI controller to be ready..." + echo " - Checking initial pod status..." + kubectl -n kube-system get pods -l app=csi-cloudscale-controller -o wide + + if ! kubectl -n kube-system rollout status statefulset/csi-cloudscale-controller --timeout=180s; then + echo "=== CSI Controller rollout FAILED ===" + echo "" + echo "--- Pod Status ---" + kubectl -n kube-system get pods -l app=csi-cloudscale-controller -o wide + echo "" + echo "--- Container States ---" + kubectl -n kube-system get pods -l app=csi-cloudscale-controller -o jsonpath='{range .items[*]}{.metadata.name}: {.status.containerStatuses[*].state}{"\n"}{end}' + echo "" + echo "--- Pod Description (includes events) ---" + kubectl -n kube-system describe pods -l app=csi-cloudscale-controller + exit 1 + fi + + echo "Waiting for CSI node daemonset to be ready..." + echo " - Checking node daemonset pod status..." + kubectl -n kube-system get pods -l app=csi-cloudscale-node -o wide + + if ! kubectl -n kube-system rollout status daemonset/csi-cloudscale-node --timeout=300s; then + echo "=== CSI Node DaemonSet rollout FAILED ===" + echo "" + echo "--- Pod Status ---" + kubectl -n kube-system get pods -l app=csi-cloudscale-node -o wide + echo "" + echo "--- Container States ---" + kubectl -n kube-system get pods -l app=csi-cloudscale-node -o jsonpath='{range .items[*]}{.metadata.name}: {.status.containerStatuses[*].state}{"\n"}{end}' + echo "" + echo "--- Pod Description (includes events) ---" + kubectl -n kube-system describe pods -l app=csi-cloudscale-node + exit 1 + fi +} + +# Deploy both CCM and CSI components +function deploy-components() { + # Deploy CCM first (required for node initialization and networking) + deploy-ccm + + # Install VolumeSnapshot CRDs and snapshot controller (required for CSI driver) + deploy-snapshot-crds + + # Import pre-built CSI image (used in CI via IMAGE_SOURCE=import) + if [[ "${IMAGE_SOURCE-build}" == "import" ]]; then + echo "Importing CSI image from image.tar..." + import-image + fi + + # Deploy CSI + deploy-csi +} + +# Execute if not sourced +if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then + + # The image name requires a slash in it, or Podman will add `localhost/` and + # confuse Kubernetes. + if [[ "$IMAGE" != *"/"* ]]; then + echo "\$IMAGE has no slash: $IMAGE" + exit 1 + fi + + first_run=$(test ! -f k8test/cluster/inventory.yml && echo "yes" || echo "no") + + ensure-k8test + ensure-cluster + + # Deploy components on both new and existing clusters. For new clusters, + # this happens after the full cluster bootstrap (including Cilium) is done. + deploy-components +fi diff --git a/helpers/test-matrix b/helpers/test-matrix new file mode 100755 index 00000000..5db5ee4f --- /dev/null +++ b/helpers/test-matrix @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Make sure the k8test/helpers/release-set CLI is available +source helpers/run-in-test-cluster +ensure-k8test > /dev/null + +# Returns a matrix entry for a specific Kubernetes version +function matrixentry { + local k8s_version="$1" + local matrix_index="$2" + + # Kubernetes release + k8test/helpers/release-set --kubernetes="$k8s_version" --limit kubernetes | jq -r '.kubernetes.version' + + # The private subnet range + echo "10.200.$matrix_index.0/24" + + # A semi-predictable cluster prefix (we want to reuse them across jobs, but + # not inside a job). Reuse allows us to ensure that the cleanup process + # at the beginning of a test will remove old runs. + echo "csi-matrix-$matrix_index" +} + +function matrixobject { + local k8s_version="$1" + local matrix_index="$2" + + matrixentry "$k8s_version" "$matrix_index" | jq --raw-input --null-input --compact-output '{ + "kubernetes": inputs, + "subnet": inputs, + "cluster_prefix": inputs, + }' +} + +function matrixentries { + # Kubernetes 1.28 (oldest supported per README compatibility matrix) + matrixobject "1.28" "1" + + # Two newest Kubernetes releases + matrixobject "-1" "2" + matrixobject "0" "3" +} + +function matrixobjects { + matrixentries | jq --slurp --compact-output '.' +} + +matrixobjects