Merge remote-tracking branch 'refs/remotes/origin/460-validation-revi…

…ew-fields-process' into 460-validation-review-fields-process

.github/actions/install-tools/action.yaml

@@ -5,4 +5,4 @@ runs:
   using: composite
   steps:
 
-    - uses: anchore/sbom-action/download-syft@251a468eed47e5082b105c3ba6ee500c0e65a764 # v0.17.6
+    - uses: anchore/sbom-action/download-syft@fc46e51fd3cb168ffb36c6d1915723c47db58abb # v0.17.7

.github/workflows/triage-label.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Add triage label
-      uses: actions/github-script@660ec11d825b714d112a6bb9727086bc2cc500b2
+      uses: actions/github-script@4020e461acd7a80762cdfff123a1fde368246fa4
       with:
         script: |
           const issueNumber = context.issue.number;

demo/simple/oscal-component-kyverno.yaml

@@ -67,6 +67,7 @@ component-definition:
                   assert:
                     all:
                     - check:
+                        (length(podsvt) > `0`): true
                         ~.podsvt:
                           metadata:
                             labels:

demo/simple/oscal-component-opa.yaml

@@ -62,6 +62,7 @@ component-definition:
               import future.keywords.every
 
               validate {
+                count(input.podsvt) > 0
                 every pod in input.podsvt {
                   podLabel := pod.metadata.labels.foo
                   podLabel == "bar"

docs/reference/domains/file-domain.md

@@ -198,3 +198,9 @@ And the following validation will confirm if the server is configured for https:
 
 ## Note on Compose
 While the file domain is capable of referencing relative file paths in the `file-spec`, Lula does not de-reference those paths during composition. If you are composing multiple files together, you must either use absolute filepaths (including network filepaths), or ensure that all referenced filepaths are relative to the output directory of the compose command. 
+
+## Evidence Collection
+
+The use of `lula dev get-resources` and `lula validate --save-resources` will produce evidence in the form of `json` files. These files provide point-in-time evidence for auditing and review purposes.
+
+Evidence collection occurs for each file specified and - in association with any error will produce an empty representation of the target file(s) data to be collected.

docs/reference/domains/kubernetes-domain.md

@@ -258,3 +258,11 @@ domain:
           type: json
           base64: true
 ```
+
+## Evidence Collection
+
+The use of `lula dev get-resources` and `lula validate --save-resources` will produce evidence in the form of `json` files. These files provide point-in-time evidence for auditing and review purposes.
+
+The Kubernetes domain requires connectivity to a cluster in order to perform data collection. The inability to connect to a cluster during the evaluation of a validation with `--save-resources` will result in an empty payload in the associated observation evidence file. 
+
+Evidence collection occurs for each resource specified and - in association with any error will produce an empty representation of the target resource(s) to be collected.

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/defenseunicorns/go-oscal v0.6.0
 	github.com/defenseunicorns/pkg/kubernetes v0.3.0
-	github.com/evertras/bubble-table v0.17.0
+	github.com/evertras/bubble-table v0.17.1
 	github.com/google/go-cmp v0.6.0
 	github.com/hashicorp/go-version v1.7.0
 	github.com/kyverno/kyverno-json v0.0.3

go.sum

@@ -141,8 +141,8 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
-github.com/evertras/bubble-table v0.17.0 h1:qQU4bi3IRxuZ5+Fvm3esyU/ucH9ufRXWhWL0fFuMn9c=
-github.com/evertras/bubble-table v0.17.0/go.mod h1:ifHujS1YxwnYSOgcR2+m3GnJ84f7CVU/4kUOxUCjEbQ=
+github.com/evertras/bubble-table v0.17.1 h1:HJwq3iQrZulXDE93ZcqJNiUVQCBbN4IJ2CkB/IxO3kk=
+github.com/evertras/bubble-table v0.17.1/go.mod h1:ifHujS1YxwnYSOgcR2+m3GnJ84f7CVU/4kUOxUCjEbQ=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=

src/cmd/dev/get-resources.go

@@ -49,12 +49,16 @@ var getResourcesCmd = &cobra.Command{
 		}
 
 		collection, err := DevGetResources(ctx, validationBytes, spinner)
+
+		// do not perform the write if there is nothing to write (likely error)
+		if collection != nil {
+			writeResources(collection, getResourcesOpts.OutputFile)
+		}
+
 		if err != nil {
 			message.Fatalf(err, "error running dev get-resources: %v", err)
 		}
 
-		writeResources(collection, getResourcesOpts.OutputFile)
-
 		spinner.Success()
 	},
 }
@@ -80,6 +84,9 @@ func DevGetResources(ctx context.Context, validationBytes []byte, spinner *messa
 		types.GetResourcesOnly(true),
 	)
 	if err != nil {
+		if lulaValidation.DomainResources != nil {
+			return *lulaValidation.DomainResources, err
+		}
 		return nil, err
 	}
 

src/pkg/domains/files/files.go

@@ -2,6 +2,7 @@ package files
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -20,6 +21,8 @@ type Domain struct {
 func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error) {
 	var workDir string
 	var ok bool
+	var errs error
+	tmpDRs := make(map[string]interface{})
 	if workDir, ok = ctx.Value(types.LulaValidationWorkDir).(string); !ok {
 		// if unset, assume lula is working in the same directory the inputFile is in
 		workDir = "."
@@ -28,6 +31,7 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
 	// see TODO below: maybe this is a REAL directory?
 	dst, err := os.MkdirTemp("", "lula-files")
 	if err != nil {
+		// allow returning on error here?
 		return nil, err
 	}
 
@@ -58,7 +62,11 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
 		file := filepath.Join(workDir, fi.Path)
 		relname, err := copyFile(dst, file)
 		if err != nil {
-			return nil, fmt.Errorf("error writing local files: %w", err)
+			// Assign empty data value for reporting purposes
+			tmpDRs[fi.Name] = map[string]interface{}{}
+			filenames[file] = fi.Name
+			errs = errors.Join(errs, fmt.Errorf("error writing local files: %w", err))
+			continue
 		}
 
 		// and save this info for later
@@ -68,23 +76,31 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
 	// get a list of all the files we just downloaded in the temporary directory
 	files, err := listFiles(dst)
 	if err != nil {
-		return nil, fmt.Errorf("error walking downloaded file tree: %w", err)
+		return tmpDRs, errors.Join(errs, fmt.Errorf("error walking downloaded file tree: %w", err))
 	}
 
 	// conftest's parser returns a map[string]interface where the filenames are
 	// the primary map keys.
+	// need to test this to understand the outcomes on a single file error on the return values
 	config, err := parser.ParseConfigurations(files)
+	// Copy values over to the temporary domain resources
+	for k, v := range config {
+		tmpDRs[k] = v
+	}
 	if err != nil {
-		return nil, err
+		errs = errors.Join(errs, err)
+		return tmpDRs, errs
 	}
 
 	// clean up the resources so it's using the filepath.Name as the map key,
 	// instead of the file path.
-	drs := make(types.DomainResources, len(config)+len(unstructuredFiles)+len(filesWithParsers))
-	for k, v := range config {
+	drs := make(types.DomainResources, len(tmpDRs)+len(unstructuredFiles)+len(filesWithParsers))
+	for k, v := range tmpDRs {
 		rel, err := filepath.Rel(dst, k)
 		if err != nil {
-			return nil, fmt.Errorf("error determining relative file path: %w", err)
+			errs = errors.Join(errs, fmt.Errorf("error determining relative file path: %w", err))
+			drs[k] = v
+			continue
 		}
 		drs[filenames[rel]] = v
 	}
@@ -95,14 +111,15 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
 		// make a sub directory by parser name
 		parserDir, err := os.MkdirTemp(dst, parserName)
 		if err != nil {
-			return nil, err
+			return drs, err
 		}
 
 		for _, fi := range filesByParser {
 			file := filepath.Join(workDir, fi.Path)
 			relname, err := copyFile(parserDir, file)
 			if err != nil {
-				return nil, fmt.Errorf("error writing local files: %w", err)
+				drs[fi.Name] = map[string]interface{}{}
+				errs = errors.Join(errs, fmt.Errorf("error writing local files: %w", err))
 			}
 
 			// and save this info for later
@@ -112,18 +129,20 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
 		// get a list of all the files we just downloaded in the temporary directory
 		files, err := listFiles(parserDir)
 		if err != nil {
-			return nil, fmt.Errorf("error walking downloaded file tree: %w", err)
+			return drs, errors.Join(errs, fmt.Errorf("error walking downloaded file tree: %w", err))
 		}
 
 		parsedConfig, err := parser.ParseConfigurationsAs(files, parserName)
 		if err != nil {
-			return nil, err
+			return drs, err
 		}
 
 		for k, v := range parsedConfig {
 			rel, err := filepath.Rel(parserDir, k)
 			if err != nil {
-				return nil, fmt.Errorf("error determining relative file path: %w", err)
+				errs = errors.Join(errs, fmt.Errorf("error determining relative file path: %w", err))
+				drs[filenames[k]] = v
+				continue
 			}
 			drs[filenames[rel]] = v
 		}
@@ -136,12 +155,14 @@ func (d Domain) GetResources(ctx context.Context) (types.DomainResources, error)
 		path := filepath.Clean(filepath.Join(workDir, f.Path))
 		b, err := os.ReadFile(path)
 		if err != nil {
-			return nil, fmt.Errorf("error reading source files: %w", err)
+			errs = errors.Join(errs, fmt.Errorf("error reading source files: %w", err))
+			drs[f.Name] = ""
+			continue
 		}
 		drs[f.Name] = string(b)
 	}
 
-	return drs, nil
+	return drs, errs
 }
 
 // IsExecutable returns false; the file domain is read-only.

src/pkg/domains/kubernetes/resource.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
 
@@ -20,28 +21,35 @@ func QueryCluster(ctx context.Context, cluster *Cluster, resources []Resource) (
 		return nil, fmt.Errorf("cluster is nil")
 	}
 
-	// We may need a new type here to hold groups of resources
-
 	collections := make(map[string]interface{}, 0)
+	var errs error
 
 	for _, resource := range resources {
 		collection, err := GetResourcesDynamically(ctx, cluster, resource.ResourceRule)
-		// log error but continue with other resources
+		// capture error but continue with other resources
 		if err != nil {
-			return nil, err
+			errs = errors.Join(errs, err)
 		}
 
-		if len(collection) > 0 {
-			// Append to collections if not empty collection
-			// convert to object if named resource
-			if resource.ResourceRule.Name != "" {
+		if resource.ResourceRule.Name != "" {
+			if len(collection) > 0 {
 				collections[resource.Name] = collection[0]
 			} else {
+				// This request returned no resources
+				collections[resource.Name] = map[string]interface{}{}
+			}
+
+		} else {
+			if len(collection) > 0 {
 				collections[resource.Name] = collection
+			} else {
+				// This request returned no resources
+				collections[resource.Name] = []map[string]interface{}{}
 			}
 		}
 	}
-	return collections, nil
+
+	return collections, errs
 }
 
 // GetResourcesDynamically() requires a dynamic interface and processes GVR to return []map[string]interface{}

src/pkg/domains/kubernetes/spec.go

@@ -95,14 +95,14 @@ func (k KubernetesDomain) GetResources(ctx context.Context) (types.DomainResourc
 
 	cluster, err := GetCluster()
 	if err != nil {
-		return nil, err
+		return resources, err
 	}
 
 	// Evaluate the create-resources parameter
 	if k.Spec.CreateResources != nil {
 		createdResources, namespaces, err = CreateAllResources(ctx, cluster, k.Spec.CreateResources)
 		if err != nil {
-			return nil, fmt.Errorf("error in create: %v", err)
+			return resources, fmt.Errorf("error in create: %v", err)
 		}
 		// Destroy the resources after everything else has been evaluated
 		defer func() {
@@ -118,20 +118,21 @@ func (k KubernetesDomain) GetResources(ctx context.Context) (types.DomainResourc
 	if k.Spec.Wait != nil {
 		err := EvaluateWait(ctx, cluster, *k.Spec.Wait)
 		if err != nil {
-			return nil, fmt.Errorf("error in wait: %v", err)
+			return resources, fmt.Errorf("error in wait: %v", err)
 		}
 	}
 
 	// Evaluate the resources parameter
 	if k.Spec.Resources != nil {
 		resources, err = QueryCluster(ctx, cluster, k.Spec.Resources)
 		if err != nil {
-			return nil, fmt.Errorf("error in query: %v", err)
+			return resources, fmt.Errorf("error in query: %v", err)
 		}
 	}
 
 	// Join the resources and createdResources
 	// Note - resource keys must be unique
+	// TODO revisit the provenance of this activity
 	if len(resources) == 0 {
 		return createdResources, nil
 	} else {

src/test/e2e/cmd/validate_test.go

@@ -37,7 +37,7 @@ func TestValidateCommand(t *testing.T) {
 		tempDir := t.TempDir()
 		outputFile := filepath.Join(tempDir, "output.yaml")
 
-		err := test(t, "-f", validInputFile, "-o", outputFile)
+		err := test(t, "-f", validInputFile, "-o", outputFile, "--save-resources")
 
 		require.NoError(t, err)
 

src/test/e2e/dev_get_resources_test.go

@@ -79,6 +79,10 @@ func TestGetResources(t *testing.T) {
 				t.Fatal("The nginx-conf resource was not found in the collection")
 			}
 
+			if len(collection["empty"].([]map[string]interface{})) != 0 {
+				t.Fatalf("expected 0 length items in the empty payload - got %v\n", len(collection["empty"].([]map[string]interface{})))
+			}
+
 			message.Info("Successfully validated dev get-resources command")
 
 			return ctx

src/test/e2e/pod_validation_test.go

@@ -345,9 +345,13 @@ func validatePodLabelFail(ctx context.Context, t *testing.T, oscalPath string) (
 
 	for _, finding := range *result.Findings {
 		state := finding.Target.Status.State
-		if state != "not-satisfied" {
-			t.Fatal("State should be not-satisfied, but got :", state)
+		if finding.Target.TargetId != "ID-3" {
+			// This validation is an empty test that should always pass unless the underlying API changes
+			if state != "not-satisfied" {
+				t.Fatal("State should be not-satisfied, but got :", state)
+			}
 		}
+
 	}
 	return result.Findings, result.Observations
 }
@@ -441,6 +445,11 @@ func validaPodResourceData(data map[string]interface{}) bool {
 				return true
 			}
 		}
+		if k == "empty" {
+			if len(v.([]interface{})) == 0 {
+				return true
+			}
+		}
 	}
 	return false
 }

src/test/e2e/scenarios/dev-get-resources/validation.yaml

@@ -17,6 +17,11 @@ domain:
           version: v1
           resource: configmaps
           namespaces: [validation-test]
+      - name: empty
+        resource-rule:
+          version: v1
+          resource: pods
+          namespaces: [doesnotexist]
 provider:
   type: opa
   opa-spec: