feat: add chainsaw scorecard test support

Makefile

@@ -89,15 +89,15 @@ build/operator-sdk build/helm-operator:
 	$(GO) build $(GO_BUILD_ARGS) -o $(BUILD_DIR)/$(@F) ./cmd/$(@F)
 
 # Build scorecard binaries.
-.PHONY: build/scorecard-test build/scorecard-test-kuttl build/custom-scorecard-tests
-build/scorecard-test build/scorecard-test-kuttl build/custom-scorecard-tests:
+.PHONY: build/scorecard-test build/scorecard-test-kuttl build/scorecard-test-chainsaw build/custom-scorecard-tests
+build/scorecard-test build/scorecard-test-kuttl build/scorecard-test-chainsaw build/custom-scorecard-tests:
 	$(GO) build $(GO_GCFLAGS) $(GO_ASMFLAGS) -o $(BUILD_DIR)/$(@F) ./images/$(@F)
 
 ##@ Dev image build
 
 # Convenience wrapper for building all remotely hosted images.
 .PHONY: image-build
-IMAGE_TARGET_LIST = operator-sdk helm-operator scorecard-test scorecard-test-kuttl
+IMAGE_TARGET_LIST = operator-sdk helm-operator scorecard-test scorecard-test-kuttl scorecard-test-chainsaw
 image-build: $(foreach i,$(IMAGE_TARGET_LIST),image/$(i)) ## Build all images.
 
 

images/scorecard-test-chainsaw/Dockerfile

@@ -0,0 +1,37 @@
+# Build the scorecard-test-chainsaw binary
+FROM --platform=$BUILDPLATFORM golang:1.21 as builder
+ARG TARGETARCH
+ARG BUILDPLATFORM
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY . .
+
+# Build
+RUN GOOS=linux GOARCH=$TARGETARCH make build/scorecard-test-chainsaw
+
+# Final image.
+FROM ghcr.io/kyverno/chainsaw:v0.1.3
+
+ENV HOME=/opt/scorecard-test-chainsaw \
+    USER_NAME=scorecard-test-chainsaw \
+    USER_UID=1001 \
+    KUBECONFIG=
+
+RUN echo "${USER_NAME}:x:${USER_UID}:0:${USER_NAME} user:${HOME}:/sbin/nologin" >> /etc/passwd
+
+WORKDIR ${HOME}
+
+COPY --from=builder /workspace/build/scorecard-test-chainsaw /usr/local/bin/scorecard-test-chainsaw
+COPY --from=builder /workspace/images/scorecard-test-chainsaw/entrypoint /usr/local/bin/entrypoint
+
+ENTRYPOINT ["/usr/local/bin/entrypoint"]
+
+USER ${USER_UID}

images/scorecard-test-chainsaw/entrypoint

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+CHAINSAW_PATH=${CHAINSAW_PATH:-"/bundle/tests/scorecard/chainsaw"}
+CHAINSAW_CONFIG=${CHAINSAW_CONFIG:-"${CHAINSAW_PATH}/.chainsaw.yaml"}
+
+chainsaw test \
+  --test-dir ${CHAINSAW_PATH} \
+  --config=${CHAINSAW_CONFIG} \
+  --report-format=JSON --report-name=/tmp/chainsaw-report > /tmp/chainsaw.stdout 2> /tmp/chainsaw.stderr
+
+# send the list of args to scorecard-test-chainsaw
+# NOTE: I'm not sure it uses the args for anything at the moment
+exec scorecard-test-chainsaw $@

images/scorecard-test-chainsaw/main.go

@@ -0,0 +1,201 @@
+// Copyright 2020 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"time"
+
+	"github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3"
+)
+
+// The scorecard test chainsaw binary processes the
+// output from chainsaw converting chainsaw output into the
+// scorecard v1alpha3.TestStatus json format.
+//
+// The chainsaw output is expected to be produced by chainsaw
+// at /tmp/chainsaw-report.json.
+func main() {
+	jsonFile, err := os.Open("/tmp/chainsaw-report.json")
+	if err != nil {
+		printErrorStatus(fmt.Errorf("could not open chainsaw report %v", err))
+		return
+	}
+	defer jsonFile.Close()
+	var byteValue []byte
+	byteValue, err = io.ReadAll(jsonFile)
+	if err != nil {
+		printErrorStatus(fmt.Errorf("could not read chainsaw report %v", err))
+		return
+	}
+	var jsonReport TestsReport
+	err = json.Unmarshal(byteValue, &jsonReport)
+	if err != nil {
+		printErrorStatus(fmt.Errorf("could not unmarshal chainsaw report %v", err))
+		return
+	}
+	if len(jsonReport.Reports) == 0 {
+		printErrorStatus(errors.New("no chainsaw test suite was found. chainsaw may not have run successfully"))
+		return
+	}
+	s := getTestStatus(jsonReport.Reports)
+	jsonOutput, err := json.MarshalIndent(s, "", "    ")
+	if err != nil {
+		printErrorStatus(fmt.Errorf("could not marshal scorecard output %v", err))
+		return
+	}
+	fmt.Println(string(jsonOutput))
+}
+
+func getTestStatus(tc []*TestReport) (s v1alpha3.TestStatus) {
+	// report the kuttl logs when kuttl tests can not be run
+	// (e.g. RBAC is not sufficient)
+	if len(tc) == 0 {
+		r := v1alpha3.TestResult{}
+		r.Log = getKuttlLogs()
+		s.Results = append(s.Results, r)
+		return s
+	}
+	for i := 0; i < len(tc); i++ {
+		r := v1alpha3.TestResult{}
+		r.Name = tc[i].Name
+		r.State = v1alpha3.PassState
+		if tc[i].Failure != nil {
+			r.State = v1alpha3.FailState
+			r.Errors = []string{tc[i].Failure.Message}
+		}
+		s.Results = append(s.Results, r)
+	}
+	return s
+}
+
+func printErrorStatus(err error) {
+	s := v1alpha3.TestStatus{}
+	r := v1alpha3.TestResult{}
+	r.State = v1alpha3.FailState
+	r.Errors = []string{err.Error()}
+	r.Log = getKuttlLogs()
+	s.Results = append(s.Results, r)
+	jsonOutput, err := json.MarshalIndent(s, "", "    ")
+	if err != nil {
+		log.Fatal(fmt.Errorf("could not marshal scorecard output %v", err))
+	}
+	fmt.Println(string(jsonOutput))
+}
+
+func getKuttlLogs() string {
+	stderrFile, err := os.ReadFile("/tmp/chainsaw.stderr")
+	if err != nil {
+		return fmt.Sprintf("could not open chainsaw stderr file: %v", err)
+	}
+	stdoutFile, err := os.ReadFile("/tmp/chainsaw.stdout")
+	if err != nil {
+		return fmt.Sprintf("could not open chainsaw stdout file: %v", err)
+	}
+	return string(stderrFile) + string(stdoutFile)
+}
+
+// chainsaw report format
+// the chainsaw structs below are copied from the kuttl master currently,
+// in the future, these structs might be pulled into SDK as
+// normal golang deps if necessary
+
+type OperationType string
+
+const (
+	OperationTypeCreate  OperationType = "create"
+	OperationTypeDelete  OperationType = "delete"
+	OperationTypeApply   OperationType = "apply"
+	OperationTypeAssert  OperationType = "assert"
+	OperationTypeError   OperationType = "error"
+	OperationTypeScript  OperationType = "script"
+	OperationTypeSleep   OperationType = "sleep"
+	OperationTypeCommand OperationType = "command"
+)
+
+// Failure represents details of a test failure.
+type Failure struct {
+	// Message provides a summary of the failure.
+	Message string `json:"message" xml:"message,attr"`
+}
+
+// TestsReport encapsulates the entire report for a test suite.
+type TestsReport struct {
+	// Name of the test suite.
+	Name string `json:"name" xml:"name,attr"`
+	// TimeStamp marks when the test suite began execution.
+	TimeStamp time.Time `json:"timestamp" xml:"timestamp,attr"`
+	// Time indicates the total duration of the test suite.
+	Time string `json:"time" xml:"time,attr"`
+	// Test count the number of tests in the files/TestReports.
+	Test int `json:"tests" xml:"tests,attr"`
+	// Reports is an array of individual test reports within this suite.
+	Reports []*TestReport `json:"testsuite" xml:"testsuite"`
+	// Failures count the number of failed tests in the suite.
+	Failures int `json:"failures" xml:"failures,attr"`
+}
+
+// TestReport represents a report for a single test.
+type TestReport struct {
+	// Name of the test.
+	Name string `json:"name" xml:"name,attr"`
+	// TimeStamp marks when the test began execution.
+	TimeStamp time.Time `json:"timestamp" xml:"timestamp,attr"`
+	// Time indicates the total duration of the test.
+	Time string `json:"time" xml:"time,attr"`
+	// Failure captures details if the test failed it should be nil otherwise.
+	Failure *Failure `json:"failure,omitempty" xml:"failure,omitempty"`
+	// Test count the number of tests in the suite/TestReport.
+	Test int `json:"tests" xml:"tests,attr"`
+	// Spec represents the specifications of the test.
+	Steps []*TestSpecStepReport `json:"testcase,omitempty" xml:"testcase,omitempty"`
+	// Concurrent indicates if the test runs concurrently with other tests.
+	Concurrent bool `json:"concurrent,omitempty" xml:"concurrent,attr,omitempty"`
+	// Namespace in which the test runs.
+	Namespace string `json:"namespace,omitempty" xml:"namespace,attr,omitempty"`
+	// Skip indicates if the test is skipped.
+	Skip bool `json:"skip,omitempty" xml:"skip,attr,omitempty"`
+	// SkipDelete indicates if resources are not deleted after test execution.
+	SkipDelete bool `json:"skipDelete,omitempty" xml:"skipDelete,attr,omitempty"`
+}
+
+// TestSpecStepReport represents a report of a single step in a test.
+type TestSpecStepReport struct {
+	// Name of the test step.
+	Name string `json:"name,omitempty" xml:"name,attr,omitempty"`
+	// Results are the outcomes of operations performed in this step.
+	Results []*OperationReport `json:"results,omitempty" xml:"results,omitempty"`
+}
+
+// OperationReport details the outcome of a single operation within a test step.
+type OperationReport struct {
+	// Name of the operation.
+	Name string `json:"name" xml:"name,attr"`
+	// TimeStamp marks when the operation began execution.
+	TimeStamp time.Time `json:"timestamp" xml:"timestamp,attr"`
+	// Time indicates the total duration of the operation.
+	Time string `json:"time" xml:"time,attr"`
+	// Result of the operation.
+	Result string `json:"result" xml:"result,attr"`
+	// Message provides additional information about the operation's outcome.
+	Message string `json:"message,omitempty" xml:"message,omitempty"`
+	// Type indicates the type of operation.
+	OperationType OperationType `json:"operationType,omitempty" xml:"operationType,attr"`
+}