Migrate from kuttl to chainsaw (#181)

* chore: migrate from kuttl to chainsaw

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* set bucket name based on namespace


* migrate vpc test from kuttl to chainsaw and remove all kuttl related tooling


---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

.github/workflows/build_test_ci.yml

@@ -161,67 +161,6 @@ jobs:
         path: .logs/*
         overwrite: true
 
-  chainsaw-test:
-    needs:  [go-build-test, docker-build]
-    runs-on: ubuntu-latest
-    steps:
-    - name: Harden Runner
-      uses: step-security/harden-runner@v2
-      with:
-        disable-sudo: true
-        egress-policy: block
-        allowed-endpoints: >
-          api.linode.com:443
-          api.github.com:443
-          github.com:443
-          gcr.io:443
-          proxy.golang.org:443
-          sum.golang.org:443
-          *.githubusercontent.com:443
-          docker.io:443
-          registry-1.docker.io:443
-          auth.docker.io:443
-          production.cloudflare.docker.com:443
-          storage.googleapis.com:443
-          registry.k8s.io:443
-          *.pkg.dev:443
-          *.amazonaws.com:443
-          *.blob.core.windows.net:443
-          quay.io:443
-          *.quay.io:443
-          api.snapcraft.io:443
-          cloud.tilt.dev:443
-
-    - uses: actions/checkout@v4
-
-    - name: Set up Go
-      uses: actions/setup-go@v5
-      with:
-        go-version-file: 'go.mod'
-        check-latest: true
-
-    - name: Docker cache
-      uses: ScribeMD/docker-cache@0.3.7
-      with:
-        key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}}
-
-    - name: Chainsaw test
-      run: make chainsaw-test
-      env:
-        GITHUB_TOKEN: ${{ secrets.github_token }}
-        LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
-
-    - name: Copy logs
-      if: ${{ always() }}
-      run: docker cp tilt-control-plane:/var/log .logs
-
-    - uses: actions/upload-artifact@v4
-      if: ${{ always() }}
-      with:
-        name: logs
-        path: .logs/*
-        overwrite: true
-
   docker-build:
     runs-on: ubuntu-latest
     steps:

Makefile

@@ -137,27 +137,18 @@ test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race -timeout 60s ./... -coverprofile cover.out
 
 .PHONY: e2etest
-e2etest:
-	make --no-print-directory _e2etest # Workaround to force the flag on Github Action
+e2etest: manifests generate local-deploy chainsaw
+	$(CHAINSAW) test ./e2e
 
-local-deploy: kind ctlptl tilt kuttl chainsaw kustomize clusterctl
+local-deploy: kind ctlptl tilt kustomize clusterctl
 	@echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode
 	$(CTLPTL) apply -f .tilt/ctlptl-config.yaml
 	$(TILT) ci --timeout 240s -f Tiltfile
 
-_e2etest: manifests generate local-deploy
-	ROOT_DIR="$(PWD)" $(KUTTL) test --config e2e/kuttl-config.yaml
-
-.PHONY: chainsaw-test
-chainsaw-test: manifests generate local-deploy
-	$(CHAINSAW) test ./e2e/linodecluster-controller ./e2e/linodemachine-controller
-
 ## --------------------------------------
 ## Build
 ## --------------------------------------
 
-##@ Build:
-
 .PHONY: build
 build: manifests generate fmt vet ## Build manager binary.
 	go build -ldflags="-X github.com/linode/cluster-api-provider-linode/version.version=$(VERSION)" -o bin/manager cmd/main.go
@@ -298,7 +289,6 @@ CLUSTERCTL     ?= $(LOCALBIN)/clusterctl
 CONTROLLER_GEN ?= $(CACHE_BIN)/controller-gen
 TILT           ?= $(LOCALBIN)/tilt
 KIND           ?= $(LOCALBIN)/kind
-KUTTL          ?= $(LOCALBIN)/kubectl-kuttl
 CHAINSAW       ?= $(CACHE_BIN)/chainsaw
 ENVTEST        ?= $(CACHE_BIN)/setup-envtest
 HUSKY          ?= $(LOCALBIN)/husky
@@ -313,15 +303,15 @@ CLUSTERCTL_VERSION       ?= v1.5.3
 CONTROLLER_TOOLS_VERSION ?= v0.14.0
 TILT_VERSION             ?= 0.33.6
 KIND_VERSION             ?= 0.20.0
-KUTTL_VERSION            ?= 0.15.0
 CHAINSAW_VERSION         ?= v0.1.7
 HUSKY_VERSION            ?= v0.2.16
 NILAWAY_VERSION          ?= latest
 GOVULNC_VERSION          ?= v1.0.1
 MOCKGEN_VERSION          ?= v0.4.0
 
 .PHONY: tools
-tools: $(KUSTOMIZE) $(CTLPTL) $(CLUSTERCTL) $(CONTROLLER_GEN) $(TILT) $(KIND) $(KUTTL) $(CHAINSAW) $(ENVTEST) $(HUSKY) $(NILAWAY) $(GOVULNC) $(MOCKGEN)
+tools: $(KUSTOMIZE) $(CTLPTL) $(CLUSTERCTL) $(CONTROLLER_GEN) $(TILT) $(KIND) $(CHAINSAW) $(ENVTEST) $(HUSKY) $(NILAWAY) $(GOVULNC) $(MOCKGEN)
+
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -360,12 +350,6 @@ $(KIND): $(LOCALBIN)
 	curl -Lso $(KIND) https://github.com/kubernetes-sigs/kind/releases/download/v$(KIND_VERSION)/kind-$(OS)-$(ARCH_SHORT)
 	chmod +x $(KIND)
 
-.PHONY: kuttl
-kuttl: $(KUTTL) ## Download kuttl locally if necessary.
-$(KUTTL): $(LOCALBIN)
-	curl -Lso $(KUTTL) https://github.com/kudobuilder/kuttl/releases/download/v$(KUTTL_VERSION)/kubectl-kuttl_$(KUTTL_VERSION)_$(OS)_$(ARCH)
-	chmod +x $(KUTTL)
-
 .PHONY: chainsaw
 chainsaw: $(CHAINSAW) ## Download chainsaw locally if necessary.
 $(CHAINSAW): $(CACHE_BIN)

cloud/services/object_storage_buckets.go

@@ -11,28 +11,22 @@ import (
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 
 	"github.com/linode/cluster-api-provider-linode/cloud/scope"
-	"github.com/linode/cluster-api-provider-linode/util"
 )
 
 func EnsureObjectStorageBucket(ctx context.Context, bScope *scope.ObjectStorageBucketScope) (*linodego.ObjectStorageBucket, error) {
-	// Buckets do not have IDs so hardcode it to 0
-	listFilter := util.Filter{
-		ID:    nil,
-		Label: *bScope.Bucket.Spec.Label,
-		Tags:  nil,
-	}
-	buckets, err := bScope.LinodeClient.ListObjectStorageBucketsInCluster(
+	bucket, err := bScope.LinodeClient.GetObjectStorageBucket(
 		ctx,
-		linodego.NewListOptions(1, listFilter.String()),
 		bScope.Bucket.Spec.Cluster,
+		*bScope.Bucket.Spec.Label,
 	)
-	if err != nil {
-		return nil, fmt.Errorf("failed to list buckets in cluster %s: %w", bScope.Bucket.Spec.Cluster, err)
+	linodeErr := &linodego.Error{}
+	if errors.As(err, linodeErr) && linodeErr.StatusCode() != http.StatusNotFound {
+		return nil, fmt.Errorf("failed to get bucket from cluster %s: %w", bScope.Bucket.Spec.Cluster, err)
 	}
-	if len(buckets) == 1 {
+	if bucket != nil {
 		bScope.Logger.Info("Bucket exists")
 
-		return &buckets[0], nil
+		return bucket, nil
 	}
 
 	opts := linodego.ObjectStorageBucketCreateOptions{
@@ -41,7 +35,6 @@ func EnsureObjectStorageBucket(ctx context.Context, bScope *scope.ObjectStorageB
 		ACL:     linodego.ACLPrivate,
 	}
 
-	var bucket *linodego.ObjectStorageBucket
 	if bucket, err = bScope.LinodeClient.CreateObjectStorageBucket(ctx, opts); err != nil {
 		return nil, fmt.Errorf("failed to create bucket: %w", err)
 	}

controller/linodeobjectstoragebucket_controller.go

@@ -54,9 +54,9 @@ type LinodeObjectStorageBucketReconciler struct {
 	ReconcileTimeout time.Duration
 }
 
-//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=linodeobjectstoragebuckets,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=linodeobjectstoragebuckets/status,verbs=get;update;patch
-//+kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=linodeobjectstoragebuckets/finalizers,verbs=update
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=linodeobjectstoragebuckets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=linodeobjectstoragebuckets/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=linodeobjectstoragebuckets/finalizers,verbs=update
 
 // +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch;create;update;patch

docs/src/developers/testing.md

@@ -0,0 +1,39 @@
+# CAPL Testing
+
+## Unit Tests
+### Running Tests
+In order to run the unit tests run the following command
+```bash
+make test
+```
+
+## E2E Tests
+For e2e tests CAPL uses the [Chainsaw project](https://kyverno.github.io/chainsaw) which leverages `kind` and `tilt` to 
+spin up a cluster with the CAPL controllers installed and then uses `chainsaw-test.yaml` files to drive e2e testing.
+
+All test live in the e2e folder with a directory structure of `e2e/${CONTROLLER_NAME}/${TEST_NAME}`
+### Running tests
+In order to run e2e tests run the following command
+```bash
+make e2etest
+```
+### Adding tests
+1. Create a new directory under the controller you are testing with the naming scheme of `e2e/${CONTROLLER_NAME}/${TEST_NAME}`
+2. Create a minimal `chainsaw-test.yaml` file in the new test dir
+    ```yaml
+   # yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+    apiVersion: chainsaw.kyverno.io/v1alpha1
+    kind: Test
+    metadata:
+      name: $TEST_NAME
+    spec:
+      template: true # set to true if you are going to use any chainsaw templating
+      steps:
+      - name: step-01
+        try:
+        - apply:
+            file: ${resource_name}.yaml
+        - assert:
+            file: 01-assert.yaml
+    ```
+3. Add any resources to create or assert on in the same directory

e2e/kind-config.yaml

@@ -1,6 +1,6 @@
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
-name: kuttl
+name: e2e
 nodes:
 - role: control-plane
   image: kindest/node:v1.28.0

e2e/linodeobjectstoragebucket-controller/minimal-linodeobjectstoragebucket/01-assert.yaml

@@ -1,11 +1,17 @@
+---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: LinodeObjectStorageBucket
 metadata:
-  name: linodeobjectstoragebucket-sample
+  name: (join('-', [$namespace, 'backups']))
 spec:
   cluster: us-sea-1
   keyGeneration: 0
 status:
   ready: true
-  keySecretName: linodeobjectstoragebucket-sample-access-keys
+  keySecretName: (join('-', [$namespace, 'backups-access-keys']))
   lastKeyGeneration: 0
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: (join('-', [$namespace, 'backups-access-keys']))

e2e/linodeobjectstoragebucket-controller/minimal-linodeobjectstoragebucket/01-create-linodeobjectstoragebucket.yaml

@@ -1,6 +1,6 @@
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: LinodeObjectStorageBucket
 metadata:
-  name: linodeobjectstoragebucket-sample
+  name: (join('-', [$namespace, 'backups']))
 spec:
   cluster: us-sea-1

e2e/linodeobjectstoragebucket-controller/minimal-linodeobjectstoragebucket/03-assert.yaml

@@ -1,11 +1,11 @@
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: LinodeObjectStorageBucket
 metadata:
-  name: linodeobjectstoragebucket-sample
+  name: (join('-', [$namespace, 'backups']))
 spec:
   cluster: us-sea-1
   keyGeneration: 1
 status:
   ready: true
-  keySecretName: linodeobjectstoragebucket-sample-access-keys
+  keySecretName: (join('-', [$namespace, 'backups-access-keys']))
   lastKeyGeneration: 1

e2e/linodeobjectstoragebucket-controller/minimal-linodeobjectstoragebucket/03-patch-bucket.yaml

@@ -1,7 +1,7 @@
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
 kind: LinodeObjectStorageBucket
 metadata:
-  name: linodeobjectstoragebucket-sample
+  name: (join('-', [$namespace, 'backups']))
 spec:
   cluster: us-sea-1
   keyGeneration: 1