update messages

Signed-off-by: Jim Bugwadia <jim@nirmata.com>

pkg/commands/scan/command_test.go

@@ -19,59 +19,59 @@ func Test_Execute(t *testing.T) {
 		out           string
 	}{{
 		name:     "foo-bar",
-		payload:  "../../../testdata/foo-bar/payload.yaml",
-		policies: []string{"../../../testdata/foo-bar/policy.yaml"},
-		out:      "../../../testdata/foo-bar/out.txt",
+		payload:  "../../../test/foo-bar/payload.yaml",
+		policies: []string{"../../../test/foo-bar/policy.yaml"},
+		out:      "../../../test/foo-bar/out.txt",
 		wantErr:  false,
 	}, {
-		name:     "jim",
-		payload:  "../../../testdata/jim/payload.json",
-		policies: []string{"../../../testdata/jim/policy.yaml"},
-		out:      "../../../testdata/jim/out.txt",
+		name:     "wildcard",
+		payload:  "../../../test/wildcard/payload.json",
+		policies: []string{"../../../test/wildcard/policy.yaml"},
+		out:      "../../../test/wildcard/out.txt",
 		wantErr:  false,
 	}, {
 		name:     "pod-no-latest",
-		payload:  "../../../testdata/pod-no-latest/payload.yaml",
-		policies: []string{"../../../testdata/pod-no-latest/policy.yaml"},
-		out:      "../../../testdata/pod-no-latest/out.txt",
+		payload:  "../../../test/pod-no-latest/payload.yaml",
+		policies: []string{"../../../test/pod-no-latest/policy.yaml"},
+		out:      "../../../test/pod-no-latest/out.txt",
 		wantErr:  false,
 	}, {
 		name:     "pod-all-latest",
-		payload:  "../../../testdata/pod-all-latest/payload.yaml",
-		policies: []string{"../../../testdata/pod-all-latest/policy.yaml"},
-		out:      "../../../testdata/pod-all-latest/out.txt",
+		payload:  "../../../test/pod-all-latest/payload.yaml",
+		policies: []string{"../../../test/pod-all-latest/policy.yaml"},
+		out:      "../../../test/pod-all-latest/out.txt",
 		wantErr:  false,
 	}, {
 		name:     "scripted",
-		payload:  "../../../testdata/scripted/payload.yaml",
-		policies: []string{"../../../testdata/scripted/policy.yaml"},
-		out:      "../../../testdata/scripted/out.txt",
+		payload:  "../../../test/scripted/payload.yaml",
+		policies: []string{"../../../test/scripted/policy.yaml"},
+		out:      "../../../test/scripted/out.txt",
 		wantErr:  false,
 	}, {
 		name:          "payload-yaml",
-		payload:       "../../../testdata/payload-yaml/payload.yaml",
+		payload:       "../../../test/payload-yaml/payload.yaml",
 		preprocessors: []string{"planned_values.root_module.resources"},
-		policies:      []string{"../../../testdata/payload-yaml/policy.yaml"},
-		out:           "../../../testdata/payload-yaml/out.txt",
+		policies:      []string{"../../../test/payload-yaml/policy.yaml"},
+		out:           "../../../test/payload-yaml/out.txt",
 		wantErr:       false,
 	}, {
 		name:          "tf-plan",
-		payload:       "../../../testdata/tf-plan/tf.plan.json",
+		payload:       "../../../test/tf-plan/payload.json",
 		preprocessors: []string{"planned_values.root_module.resources"},
-		policies:      []string{"../../../testdata/tf-plan/policy.yaml"},
-		out:           "../../../testdata/tf-plan/out.txt",
+		policies:      []string{"../../../test/tf-plan/policy.yaml"},
+		out:           "../../../test/tf-plan/out.txt",
 		wantErr:       false,
 	}, {
 		name:     "escaped",
-		payload:  "../../../testdata/escaped/payload.yaml",
-		policies: []string{"../../../testdata/escaped/policy.yaml"},
-		out:      "../../../testdata/escaped/out.txt",
+		payload:  "../../../test/escaped/payload.yaml",
+		policies: []string{"../../../test/escaped/policy.yaml"},
+		out:      "../../../test/escaped/out.txt",
 		wantErr:  false,
 	}, {
 		name:     "dockerfile",
-		payload:  "../../../testdata/dockerfile/input.json",
-		policies: []string{"../../../testdata/dockerfile/policy-check-external.yaml"},
-		out:      "../../../testdata/dockerfile/out.txt",
+		payload:  "../../../test/dockerfile/payload.json",
+		policies: []string{"../../../test/dockerfile/policy.yaml"},
+		out:      "../../../test/dockerfile/out.txt",
 		wantErr:  false,
 	}}
 	for _, tt := range tests {

pkg/engine/assert/match.go

@@ -19,46 +19,48 @@ func MatchAssert(ctx context.Context, path *field.Path, match *v1alpha1.Assert,
 		return nil, field.Invalid(path, match, "an empty assert is not valid")
 	} else {
 		if len(match.Any) != 0 {
-			var errs []error
+			var fails []error
 			path := path.Child("any")
 			for i, assertion := range match.Any {
-				_errs, err := validate(ctx, path.Index(i).Child("check"), assertion.Check.Value, actual, bindings)
+				checkFails, err := validate(ctx, path.Index(i).Child("check"), assertion.Check.Value, actual, bindings)
 				if err != nil {
-					return errs, err
+					return fails, err
 				}
-				if len(_errs) == 0 {
-					errs = nil
+				if len(checkFails) == 0 {
+					fails = nil
 					break
 				}
 				if assertion.Message != "" {
-					errs = append(errs, errors.New(template.String(ctx, assertion.Message, actual, bindings)))
+					msg := template.String(ctx, assertion.Message, actual, bindings)
+					msg += ": " + checkFails.ToAggregate().Error()
+					fails = append(fails, errors.New(msg))
 				} else {
-					for _, err := range _errs {
-						errs = append(errs, err)
-					}
+					fails = append(fails, checkFails.ToAggregate())
 				}
 			}
-			if errs != nil {
-				return errs, nil
+			if fails != nil {
+				return fails, nil
 			}
 		}
 		if len(match.All) != 0 {
-			var errs []error
+			var fails []error
 			path := path.Child("all")
 			for i, assertion := range match.All {
-				_errs, err := validate(ctx, path.Index(i).Child("check"), assertion.Check.Value, actual, bindings)
+				checkFails, err := validate(ctx, path.Index(i).Child("check"), assertion.Check.Value, actual, bindings)
 				if err != nil {
-					return errs, err
+					return fails, err
 				}
-				if assertion.Message != "" {
-					errs = append(errs, errors.New(template.String(ctx, assertion.Message, actual, bindings)))
-				} else {
-					for _, err := range _errs {
-						errs = append(errs, err)
+				if len(checkFails) > 0 {
+					if assertion.Message != "" {
+						msg := template.String(ctx, assertion.Message, actual, bindings)
+						msg += ": " + checkFails.ToAggregate().Error()
+						fails = append(fails, errors.New(msg))
+					} else {
+						fails = append(fails, checkFails.ToAggregate())
 					}
 				}
 			}
-			return errs, nil
+			return fails, nil
 		}
 		return nil, nil
 	}

pkg/engine/assert/parse.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"reflect"
 
-	"github.com/jmespath-community/go-jmespath/pkg/binding"
 	jpbinding "github.com/jmespath-community/go-jmespath/pkg/binding"
 	"github.com/kyverno/kyverno-json/pkg/engine/match"
 	"github.com/kyverno/kyverno-json/pkg/engine/template"
@@ -97,7 +96,7 @@ func (n mapNode) assert(ctx context.Context, path *field.Path, value interface{}
 // if lengths match all descendants are evaluated with their corresponding items.
 type sliceNode []Assertion
 
-func (n sliceNode) assert(ctx context.Context, path *field.Path, value interface{}, bindings binding.Bindings) (field.ErrorList, error) {
+func (n sliceNode) assert(ctx context.Context, path *field.Path, value interface{}, bindings jpbinding.Bindings) (field.ErrorList, error) {
 	var errs field.ErrorList
 	if reflectutils.GetKind(value) != reflect.Slice {
 		return nil, field.TypeInvalid(path, value, "expected a slice")
@@ -125,7 +124,7 @@ type scalarNode struct {
 	rhs interface{}
 }
 
-func (n *scalarNode) assert(ctx context.Context, path *field.Path, value interface{}, bindings binding.Bindings) (field.ErrorList, error) {
+func (n *scalarNode) assert(ctx context.Context, path *field.Path, value interface{}, bindings jpbinding.Bindings) (field.ErrorList, error) {
 	rhs := n.rhs
 	expression := parseExpression(ctx, rhs)
 	// we only project if the expression uses the engine syntax

pkg/json-engine/engine.go

@@ -65,11 +65,7 @@ func New() engine.Engine[JsonEngineRequest, JsonEngineResponse] {
 			if err != nil {
 				response.Failure = err
 			} else if err := multierr.Combine(errs...); err != nil {
-				// if r.rule.Validation.Message != "" {
-				// 	response.Error = errors.New(template.String(ctx, r.rule.Validation.Message, r.value, r.bindings))
-				// } else {
 				response.Error = err
-				// }
 			}
 			return response
 		}).

pkg/utils/rest/rest_test.go

@@ -14,7 +14,7 @@ func TestRestConfig(t *testing.T) {
 		wantErr    bool
 	}{{
 		name:       "empty",
-		kubeConfig: "../../../testdata/.kube/config",
+		kubeConfig: "../../../test/.kube/config",
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

test/dockerfile/out.txt

@@ -0,0 +1,6 @@
+Loading policies ...
+Loading payload ...
+Pre processing ...
+Running ( evaluating 1 resource against 1 policy ) ...
+- check-dockerfile / no-external /  FAILED: HTTP calls are not allowed: all[0].check.~.(Stages[].Commands[].Args[].Value)[0].(contains(@, 'https://') || contains(@, 'http://')): Invalid value: true: Expected value: false; wget is not allowed: all[3].check.~.(Stages[].Commands[].CmdLine[])[0].(contains(@, 'wget')): Invalid value: true: Expected value: false
+Done

test/dockerfile/policy.yaml

@@ -0,0 +1,26 @@
+apiVersion: json.kyverno.io/v1alpha1
+kind: Policy
+metadata:
+  name: check-dockerfile
+spec:
+  rules:
+    - name: no-external
+      validate:
+        assert:
+          all:
+          - message: "HTTP calls are not allowed"
+            check:
+              ~.(Stages[].Commands[].Args[].Value):
+                (contains(@, 'https://') || contains(@, 'http://')): false
+          - message: "HTTP calls are not allowed"
+            check:
+              ~.(Stages[].Commands[].CmdLine[]):
+                (contains(@, 'https://') || contains(@, 'http://')): false
+          - message: "curl is not allowed"
+            check:
+              ~.(Stages[].Commands[].CmdLine[]):
+                (contains(@, 'curl')): false
+          - message: "wget is not allowed"
+            check:
+              ~.(Stages[].Commands[].CmdLine[]):
+                (contains(@, 'wget')): false

testdata/tf-plan/out.txt → test/payload-yaml/out.txt

@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED: Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}
+- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED: Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}: all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"}
 Done

test/pod-no-latest/out.txt

@@ -0,0 +1,6 @@
+Loading policies ...
+Loading payload ...
+Pre processing ...
+Running ( evaluating 1 resource against 1 policy ) ...
+- test / pod-no-latest /  FAILED: [all[0].check.spec.~foo.containers->foos[0].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false, all[0].check.spec.~foo.containers->foos[1].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false, all[0].check.spec.~foo.containers->foos[2].(at($foos, $foo).image)->foo.(ends_with($foo, $tag)): Invalid value: true: Expected value: false]; [all[1].check.spec.~.containers->foo[0].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false, all[1].check.spec.~.containers->foo[1].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false, all[1].check.spec.~.containers->foo[2].image.(ends_with(@, ':latest')): Invalid value: true: Expected value: false]; [all[2].check.~index.(spec.containers[*].image)->images[0].(ends_with(@, ':latest')): Invalid value: true: Expected value: false, all[2].check.~index.(spec.containers[*].image)->images[1].(ends_with(@, ':latest')): Invalid value: true: Expected value: false, all[2].check.~index.(spec.containers[*].image)->images[2].(ends_with(@, ':latest')): Invalid value: true: Expected value: false]
+Done

testdata/payload-yaml/out.txt → test/tf-plan/out.txt

@@ -2,5 +2,5 @@ Loading policies ...
 Loading payload ...
 Pre processing ...
 Running ( evaluating 1 resource against 1 policy ) ...
-- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED: Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}
+- required-s3-tags / require-team-tag / aws_s3_bucket.example FAILED: Bucket `example` (aws_s3_bucket.example) does not have the required tags {"Team":"Kyverno"}: all[0].check.values.tags: Invalid value: map[string]interface {}{"Environment":"Dev", "Name":"My bucket"}: Expected value: map[string]interface {}{"Team":"Kyverno"}
 Done