From 25dbe0651f15535f236f2e91fb566f2f28bb2670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Tue, 24 Sep 2024 10:25:24 +0200 Subject: [PATCH] refactor: policy compilation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché --- go.mod | 3 +- go.sum | 2 - pkg/apis/policy/v1alpha1/any.go | 10 +- pkg/apis/policy/v1alpha1/assertion_tree.go | 10 +- pkg/commands/scan/command_test.go | 222 ++++++------ pkg/json-engine/compiler.go | 394 +++++++++++++++++++++ pkg/json-engine/engine.go | 188 +--------- pkg/json-engine/model.go | 78 ++++ pkg/matching/compiler.go | 52 --- pkg/matching/match.go | 161 --------- 10 files changed, 607 insertions(+), 513 deletions(-) create mode 100644 pkg/json-engine/compiler.go create mode 100644 pkg/json-engine/model.go delete mode 100644 pkg/matching/compiler.go delete mode 100644 pkg/matching/match.go diff --git a/go.mod b/go.mod index 63f3e25b..91e64778 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,6 @@ go 1.22.2 require ( github.com/aquilax/truncate v1.0.0 github.com/blang/semver/v4 v4.0.0 - github.com/cespare/xxhash/v2 v2.3.0 - github.com/elastic/go-freelru v0.13.0 github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 github.com/google/cel-go v0.20.1 @@ -38,6 +36,7 @@ require ( github.com/bytedance/sonic v1.12.2 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect diff --git a/go.sum b/go.sum index 0ea8258f..b775931f 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 h1:S6Dco8FtAhEI/qkg/00H6RdEGC+MCy5GPiQ+xweNRFE= github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc= -github.com/elastic/go-freelru v0.13.0 h1:TKKY6yCfNNNky7Pj9xZAOEpBcdNgZJfihEftOb55omg= -github.com/elastic/go-freelru v0.13.0/go.mod h1:bSdWT4M0lW79K8QbX6XY2heQYSCqD7THoYf82pT/H3I= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= diff --git a/pkg/apis/policy/v1alpha1/any.go b/pkg/apis/policy/v1alpha1/any.go index bea3c96d..cf7c89d1 100644 --- a/pkg/apis/policy/v1alpha1/any.go +++ b/pkg/apis/policy/v1alpha1/any.go @@ -1,8 +1,8 @@ package v1alpha1 import ( + "github.com/kyverno/kyverno-json/pkg/core/compilers" "github.com/kyverno/kyverno-json/pkg/core/projection" - hashutils "github.com/kyverno/kyverno-json/pkg/utils/hash" "k8s.io/apimachinery/pkg/util/json" ) @@ -12,18 +12,16 @@ import ( // +kubebuilder:validation:Type:="" type Any struct { _value any - _hash string } func NewAny(value any) Any { return Any{ _value: value, - _hash: hashutils.Hash(value), } } -func (t *Any) Compile(compiler func(string, any, string) (projection.ScalarHandler, error), defaultCompiler string) (projection.ScalarHandler, error) { - return compiler(t._hash, t._value, defaultCompiler) +func (t *Any) Compile(compilers compilers.Compilers) (projection.ScalarHandler, error) { + return projection.ParseScalar(t._value, compilers) } func (a *Any) MarshalJSON() ([]byte, error) { @@ -37,13 +35,11 @@ func (a *Any) UnmarshalJSON(data []byte) error { return err } a._value = v - a._hash = hashutils.Hash(a._value) return nil } func (in *Any) DeepCopyInto(out *Any) { out._value = deepCopy(in._value) - out._hash = in._hash } func (in *Any) DeepCopy() *Any { diff --git a/pkg/apis/policy/v1alpha1/assertion_tree.go b/pkg/apis/policy/v1alpha1/assertion_tree.go index 24e763ac..79efd8d8 100644 --- a/pkg/apis/policy/v1alpha1/assertion_tree.go +++ b/pkg/apis/policy/v1alpha1/assertion_tree.go @@ -2,7 +2,7 @@ package v1alpha1 import ( "github.com/kyverno/kyverno-json/pkg/core/assertion" - hashutils "github.com/kyverno/kyverno-json/pkg/utils/hash" + "github.com/kyverno/kyverno-json/pkg/core/compilers" "k8s.io/apimachinery/pkg/util/json" ) @@ -12,18 +12,16 @@ import ( // AssertionTree represents an assertion tree. type AssertionTree struct { _tree any - _hash string } func NewAssertionTree(value any) AssertionTree { return AssertionTree{ _tree: value, - _hash: hashutils.Hash(value), } } -func (t *AssertionTree) Compile(compiler func(string, any, string) (assertion.Assertion, error), defaultCompiler string) (assertion.Assertion, error) { - return compiler(t._hash, t._tree, defaultCompiler) +func (t *AssertionTree) Compile(compilers compilers.Compilers) (assertion.Assertion, error) { + return assertion.Parse(t._tree, compilers) } func (a *AssertionTree) MarshalJSON() ([]byte, error) { @@ -37,11 +35,9 @@ func (a *AssertionTree) UnmarshalJSON(data []byte) error { return err } a._tree = v - a._hash = hashutils.Hash(a._tree) return nil } func (in *AssertionTree) DeepCopyInto(out *AssertionTree) { out._tree = deepCopy(in._tree) - out._hash = in._hash } diff --git a/pkg/commands/scan/command_test.go b/pkg/commands/scan/command_test.go index f4e8a5f6..efd738a1 100644 --- a/pkg/commands/scan/command_test.go +++ b/pkg/commands/scan/command_test.go @@ -19,122 +19,122 @@ func Test_Execute(t *testing.T) { wantErr bool out string }{{ - // name: "foo-bar", - // payload: "../../../test/commands/scan/foo-bar/payload.yaml", - // policies: []string{"../../../test/commands/scan/foo-bar/policy.yaml"}, - // out: "../../../test/commands/scan/foo-bar/out.txt", - // wantErr: false, - // }, { + name: "foo-bar", + payload: "../../../test/commands/scan/foo-bar/payload.yaml", + policies: []string{"../../../test/commands/scan/foo-bar/policy.yaml"}, + out: "../../../test/commands/scan/foo-bar/out.txt", + wantErr: false, + }, { name: "cel", payload: "../../../test/commands/scan/cel/payload.yaml", policies: []string{"../../../test/commands/scan/cel/policy.yaml"}, out: "../../../test/commands/scan/cel/out.txt", wantErr: false, - // }, { - // name: "wildcard", - // payload: "../../../test/commands/scan/wildcard/payload.json", - // policies: []string{"../../../test/commands/scan/wildcard/policy.yaml"}, - // out: "../../../test/commands/scan/wildcard/out.txt", - // wantErr: false, - // }, { - // name: "bindings", - // bindings: "../../../test/commands/scan/bindings/bindings.yaml", - // payload: "../../../test/commands/scan/bindings/payload.yaml", - // policies: []string{"../../../test/commands/scan/bindings/policy.yaml"}, - // out: "../../../test/commands/scan/bindings/out.txt", - // wantErr: false, - // }, { - // name: "pod-no-latest", - // payload: "../../../test/commands/scan/pod-no-latest/payload.yaml", - // policies: []string{"../../../test/commands/scan/pod-no-latest/policy.yaml"}, - // out: "../../../test/commands/scan/pod-no-latest/out.txt", - // wantErr: false, - // }, { - // name: "pod-all-latest", - // payload: "../../../test/commands/scan/pod-all-latest/payload.yaml", - // policies: []string{"../../../test/commands/scan/pod-all-latest/policy.yaml"}, - // out: "../../../test/commands/scan/pod-all-latest/out.txt", - // wantErr: false, - // }, { - // name: "scripted", - // payload: "../../../test/commands/scan/scripted/payload.yaml", - // policies: []string{"../../../test/commands/scan/scripted/policy.yaml"}, - // out: "../../../test/commands/scan/scripted/out.txt", - // wantErr: false, - // }, { - // name: "payload-yaml", - // payload: "../../../test/commands/scan/payload-yaml/payload.yaml", - // preprocessors: []string{"planned_values.root_module.resources"}, - // policies: []string{"../../../test/commands/scan/payload-yaml/policy.yaml"}, - // out: "../../../test/commands/scan/payload-yaml/out.txt", - // wantErr: false, - // }, { - // name: "tf-plan", - // payload: "../../../test/commands/scan/tf-plan/payload.json", - // preprocessors: []string{"planned_values.root_module.resources"}, - // policies: []string{"../../../test/commands/scan/tf-plan/policy.yaml"}, - // out: "../../../test/commands/scan/tf-plan/out.txt", - // wantErr: false, - // }, { - // name: "escaped", - // payload: "../../../test/commands/scan/escaped/payload.yaml", - // policies: []string{"../../../test/commands/scan/escaped/policy.yaml"}, - // out: "../../../test/commands/scan/escaped/out.txt", - // wantErr: false, - // }, { - // name: "dockerfile", - // payload: "../../../test/commands/scan/dockerfile/payload.json", - // policies: []string{"../../../test/commands/scan/dockerfile/policy.yaml"}, - // out: "../../../test/commands/scan/dockerfile/out.txt", - // wantErr: false, - // }, { - // name: "tf-s3", - // payload: "../../../test/commands/scan/tf-s3/payload.json", - // policies: []string{"../../../test/commands/scan/tf-s3/policy.yaml"}, - // out: "../../../test/commands/scan/tf-s3/out.txt", - // wantErr: false, - // }, { - // name: "tf-ec2", - // payload: "../../../test/commands/scan/tf-ec2/payload.json", - // preprocessors: []string{"planned_values.root_module.resources"}, - // policies: []string{"../../../test/commands/scan/tf-ec2/policy.yaml"}, - // out: "../../../test/commands/scan/tf-ec2/out.txt", - // wantErr: false, - // }, { - // name: "tf-ecs-cluster-1", - // payload: "../../../test/commands/scan/tf-ecs-cluster/payload.json", - // preprocessors: []string{"planned_values.root_module.resources"}, - // policies: []string{"../../../test/commands/scan/tf-ecs-cluster/01-policy.yaml"}, - // out: "../../../test/commands/scan/tf-ecs-cluster/01-out.txt", - // wantErr: false, - // }, { - // name: "tf-ecs-cluster-2", - // payload: "../../../test/commands/scan/tf-ecs-cluster/payload.json", - // preprocessors: []string{"planned_values.root_module.resources"}, - // policies: []string{"../../../test/commands/scan/tf-ecs-cluster/02-policy.yaml"}, - // out: "../../../test/commands/scan/tf-ecs-cluster/02-out.txt", - // wantErr: false, - // }, { - // name: "tf-ecs-service-1", - // payload: "../../../test/commands/scan/tf-ecs-service/payload.json", - // preprocessors: []string{"planned_values.root_module.resources"}, - // policies: []string{"../../../test/commands/scan/tf-ecs-service/01-policy.yaml"}, - // out: "../../../test/commands/scan/tf-ecs-service/01-out.txt", - // wantErr: false, - // }, { - // name: "tf-ecs-service-2", - // payload: "../../../test/commands/scan/tf-ecs-service/payload.json", - // preprocessors: []string{"planned_values.root_module.resources"}, - // policies: []string{"../../../test/commands/scan/tf-ecs-service/02-policy.yaml"}, - // out: "../../../test/commands/scan/tf-ecs-service/02-out.txt", - // wantErr: false, - // }, { - // name: "tf-ecs-task-definition", - // payload: "../../../test/commands/scan/tf-ecs-task-definition/payload.json", - // preprocessors: []string{"planned_values.root_module.resources"}, - // policies: []string{"../../../test/commands/scan/tf-ecs-task-definition/policy.yaml"}, - // out: "../../../test/commands/scan/tf-ecs-task-definition/out.txt", - // wantErr: false, + }, { + name: "wildcard", + payload: "../../../test/commands/scan/wildcard/payload.json", + policies: []string{"../../../test/commands/scan/wildcard/policy.yaml"}, + out: "../../../test/commands/scan/wildcard/out.txt", + wantErr: false, + }, { + name: "bindings", + bindings: "../../../test/commands/scan/bindings/bindings.yaml", + payload: "../../../test/commands/scan/bindings/payload.yaml", + policies: []string{"../../../test/commands/scan/bindings/policy.yaml"}, + out: "../../../test/commands/scan/bindings/out.txt", + wantErr: false, + }, { + name: "pod-no-latest", + payload: "../../../test/commands/scan/pod-no-latest/payload.yaml", + policies: []string{"../../../test/commands/scan/pod-no-latest/policy.yaml"}, + out: "../../../test/commands/scan/pod-no-latest/out.txt", + wantErr: false, + }, { + name: "pod-all-latest", + payload: "../../../test/commands/scan/pod-all-latest/payload.yaml", + policies: []string{"../../../test/commands/scan/pod-all-latest/policy.yaml"}, + out: "../../../test/commands/scan/pod-all-latest/out.txt", + wantErr: false, + }, { + name: "scripted", + payload: "../../../test/commands/scan/scripted/payload.yaml", + policies: []string{"../../../test/commands/scan/scripted/policy.yaml"}, + out: "../../../test/commands/scan/scripted/out.txt", + wantErr: false, + }, { + name: "payload-yaml", + payload: "../../../test/commands/scan/payload-yaml/payload.yaml", + preprocessors: []string{"planned_values.root_module.resources"}, + policies: []string{"../../../test/commands/scan/payload-yaml/policy.yaml"}, + out: "../../../test/commands/scan/payload-yaml/out.txt", + wantErr: false, + }, { + name: "tf-plan", + payload: "../../../test/commands/scan/tf-plan/payload.json", + preprocessors: []string{"planned_values.root_module.resources"}, + policies: []string{"../../../test/commands/scan/tf-plan/policy.yaml"}, + out: "../../../test/commands/scan/tf-plan/out.txt", + wantErr: false, + }, { + name: "escaped", + payload: "../../../test/commands/scan/escaped/payload.yaml", + policies: []string{"../../../test/commands/scan/escaped/policy.yaml"}, + out: "../../../test/commands/scan/escaped/out.txt", + wantErr: false, + }, { + name: "dockerfile", + payload: "../../../test/commands/scan/dockerfile/payload.json", + policies: []string{"../../../test/commands/scan/dockerfile/policy.yaml"}, + out: "../../../test/commands/scan/dockerfile/out.txt", + wantErr: false, + }, { + name: "tf-s3", + payload: "../../../test/commands/scan/tf-s3/payload.json", + policies: []string{"../../../test/commands/scan/tf-s3/policy.yaml"}, + out: "../../../test/commands/scan/tf-s3/out.txt", + wantErr: false, + }, { + name: "tf-ec2", + payload: "../../../test/commands/scan/tf-ec2/payload.json", + preprocessors: []string{"planned_values.root_module.resources"}, + policies: []string{"../../../test/commands/scan/tf-ec2/policy.yaml"}, + out: "../../../test/commands/scan/tf-ec2/out.txt", + wantErr: false, + }, { + name: "tf-ecs-cluster-1", + payload: "../../../test/commands/scan/tf-ecs-cluster/payload.json", + preprocessors: []string{"planned_values.root_module.resources"}, + policies: []string{"../../../test/commands/scan/tf-ecs-cluster/01-policy.yaml"}, + out: "../../../test/commands/scan/tf-ecs-cluster/01-out.txt", + wantErr: false, + }, { + name: "tf-ecs-cluster-2", + payload: "../../../test/commands/scan/tf-ecs-cluster/payload.json", + preprocessors: []string{"planned_values.root_module.resources"}, + policies: []string{"../../../test/commands/scan/tf-ecs-cluster/02-policy.yaml"}, + out: "../../../test/commands/scan/tf-ecs-cluster/02-out.txt", + wantErr: false, + }, { + name: "tf-ecs-service-1", + payload: "../../../test/commands/scan/tf-ecs-service/payload.json", + preprocessors: []string{"planned_values.root_module.resources"}, + policies: []string{"../../../test/commands/scan/tf-ecs-service/01-policy.yaml"}, + out: "../../../test/commands/scan/tf-ecs-service/01-out.txt", + wantErr: false, + }, { + name: "tf-ecs-service-2", + payload: "../../../test/commands/scan/tf-ecs-service/payload.json", + preprocessors: []string{"planned_values.root_module.resources"}, + policies: []string{"../../../test/commands/scan/tf-ecs-service/02-policy.yaml"}, + out: "../../../test/commands/scan/tf-ecs-service/02-out.txt", + wantErr: false, + }, { + name: "tf-ecs-task-definition", + payload: "../../../test/commands/scan/tf-ecs-task-definition/payload.json", + preprocessors: []string{"planned_values.root_module.resources"}, + policies: []string{"../../../test/commands/scan/tf-ecs-task-definition/policy.yaml"}, + out: "../../../test/commands/scan/tf-ecs-task-definition/out.txt", + wantErr: false, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/json-engine/compiler.go b/pkg/json-engine/compiler.go new file mode 100644 index 00000000..b5a7ec6c --- /dev/null +++ b/pkg/json-engine/compiler.go @@ -0,0 +1,394 @@ +package jsonengine + +import ( + "fmt" + "sync" + "time" + + "github.com/jmespath-community/go-jmespath/pkg/binding" + "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" + "github.com/kyverno/kyverno-json/pkg/core/compilers" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +type compiler struct{} + +func (c *compiler) compileContextEntry( + path *field.Path, + compilers compilers.Compilers, + in v1alpha1.ContextEntry, +) (func(any, binding.Bindings) binding.Bindings, error) { + if in.Compiler != nil { + compilers = compilers.WithDefaultCompiler(string(*in.Compiler)) + } + handler, err := in.Variable.Compile(compilers) + if err != nil { + return nil, field.InternalError(path.Child("variable"), err) + } + return func(resource any, bindings binding.Bindings) binding.Bindings { + binding := binding.NewDelegate( + sync.OnceValues( + func() (any, error) { + projected, err := handler(resource, bindings) + if err != nil { + return nil, field.InternalError(path.Child("variable"), err) + } + return projected, nil + }, + ), + ) + return bindings.Register("$"+in.Name, binding) + }, nil +} + +func (c *compiler) compileContext( + path *field.Path, + compilers compilers.Compilers, + in ...v1alpha1.ContextEntry, +) (func(any, binding.Bindings) binding.Bindings, error) { + var out []func(any, binding.Bindings) binding.Bindings + for i, entry := range in { + entry, err := c.compileContextEntry(path.Index(i), compilers, entry) + if err != nil { + return nil, err + } + out = append(out, entry) + } + return func(resource any, bindings binding.Bindings) binding.Bindings { + for _, entry := range out { + bindings = entry(resource, bindings) + } + return bindings + }, nil +} + +func (c *compiler) compileMatch( + path *field.Path, + compilers compilers.Compilers, + in *v1alpha1.Match, +) (func(any, binding.Bindings) (field.ErrorList, error), error) { + if in == nil { + return nil, nil + } + if len(in.Any) == 0 && len(in.All) == 0 { + return nil, field.Invalid(path, in, "an empty match is not valid") + } + if in.Compiler != nil { + compilers = compilers.WithDefaultCompiler(string(*in.Compiler)) + } + _any, err := c.compileAssertionTrees(path.Child("any"), compilers, in.Any...) + if err != nil { + return nil, err + } + _all, err := c.compileAssertionTrees(path.Child("all"), compilers, in.All...) + if err != nil { + return nil, err + } + return func(resource any, bindings binding.Bindings) (field.ErrorList, error) { + var errs field.ErrorList + for _, assertion := range _any { + _errs, err := assertion(resource, bindings) + if err != nil { + return errs, err + } + if len(_errs) == 0 { + return nil, nil + } + errs = append(errs, _errs...) + } + for _, assertion := range _all { + _errs, err := assertion(resource, bindings) + if err != nil { + return errs, err + } + errs = append(errs, _errs...) + } + return errs, nil + }, nil +} + +func (c *compiler) compileAssert( + path *field.Path, + compilers compilers.Compilers, + in v1alpha1.Assert, +) (func(any, binding.Bindings) (Results, error), error) { + if in.Compiler != nil { + compilers = compilers.WithDefaultCompiler(string(*in.Compiler)) + } + if len(in.Any) == 0 && len(in.All) == 0 { + return nil, field.Invalid(path, in, "an empty assert is not valid") + } + _any, err := c.compileAssertions(path.Child("any"), compilers, in.Any...) + if err != nil { + return nil, err + } + _all, err := c.compileAssertions(path.Child("all"), compilers, in.All...) + if err != nil { + return nil, err + } + return func(resource any, bindings binding.Bindings) (Results, error) { + if len(_any) != 0 { + var fails Results + for _, assertion := range _any { + result, err := assertion(resource, bindings) + if err != nil { + return fails, err + } + if len(result.ErrorList) == 0 { + fails = nil + break + } + fails = append(fails, result) + } + if fails != nil { + return fails, nil + } + } + if len(_all) != 0 { + var fails Results + for _, assertion := range _all { + result, err := assertion(resource, bindings) + if err != nil { + return fails, err + } + if len(result.ErrorList) > 0 { + fails = append(fails, result) + } + } + return fails, nil + } + return nil, nil + }, nil +} + +func (c *compiler) compileAssertions( + path *field.Path, + compilers compilers.Compilers, + in ...v1alpha1.Assertion, +) ([]func(any, binding.Bindings) (Result, error), error) { + var out []func(any, binding.Bindings) (Result, error) + for i, in := range in { + if in, err := c.compileAssertion(path.Index(i), compilers, in); err != nil { + return nil, err + } else { + out = append(out, in) + } + } + return out, nil +} + +func (c *compiler) compileAssertion( + path *field.Path, + compilers compilers.Compilers, + in v1alpha1.Assertion, +) (func(any, binding.Bindings) (Result, error), error) { + if in.Compiler != nil { + compilers = compilers.WithDefaultCompiler(string(*in.Compiler)) + } + check, err := c.compileAssertionTree(path.Child("check"), compilers, in.Check) + if err != nil { + return nil, err + } + return func(resource any, bindings binding.Bindings) (Result, error) { + var result Result + errs, err := check(resource, bindings) + if len(errs) != 0 { + result.ErrorList = errs + if in.Message != nil { + result.Message = in.Message.Format(resource, bindings, compilers.Jp.Options()...) + } + } + return result, err + }, nil +} + +func (c *compiler) compileAssertionTrees( + path *field.Path, + compilers compilers.Compilers, + in ...v1alpha1.AssertionTree, +) ([]func(any, binding.Bindings) (field.ErrorList, error), error) { + var out []func(any, binding.Bindings) (field.ErrorList, error) + for i, in := range in { + if in, err := c.compileAssertionTree(path.Index(i), compilers, in); err != nil { + return nil, err + } else { + out = append(out, in) + } + } + return out, nil +} + +func (c *compiler) compileAssertionTree( + path *field.Path, + compilers compilers.Compilers, + in v1alpha1.AssertionTree, +) (func(any, binding.Bindings) (field.ErrorList, error), error) { + check, err := in.Compile(compilers) + if err != nil { + return nil, err + } + return func(resource any, bindings binding.Bindings) (field.ErrorList, error) { + return check.Assert(path, resource, bindings) + }, nil +} + +func (c *compiler) compileIdentifier( + _ *field.Path, + compilers compilers.Compilers, + in string, +) (func(any, binding.Bindings) string, error) { + if in == "" { + return func(resource any, bindings binding.Bindings) string { + return "" + }, nil + } + program, err := compilers.Jp.Compile(in) + if err != nil { + return nil, err + } + return func(resource any, bindings binding.Bindings) string { + result, err := program(resource, bindings) + if err != nil { + return fmt.Sprintf("(error: %s)", err) + } else { + return fmt.Sprint(result) + } + }, nil +} + +func (c *compiler) compileFeedbacks( + path *field.Path, + compilers compilers.Compilers, + in ...v1alpha1.Feedback, +) (func(any, binding.Bindings) map[string]Feedback, error) { + if len(in) == 0 { + return func(any, binding.Bindings) map[string]Feedback { + return nil + }, nil + } + feedback := map[string]func(any, binding.Bindings) Feedback{} + for i, in := range in { + f, err := c.compileFeedback(path.Index(i), compilers, in) + if err != nil { + return nil, err + } + feedback[in.Name] = f + } + return func(resource any, bindings binding.Bindings) map[string]Feedback { + out := map[string]Feedback{} + for name, f := range feedback { + out[name] = f(resource, bindings) + } + return out + }, nil +} + +func (c *compiler) compileFeedback( + _ *field.Path, + compilers compilers.Compilers, + in v1alpha1.Feedback, +) (func(any, binding.Bindings) Feedback, error) { + if in.Compiler != nil { + compilers = compilers.WithDefaultCompiler(string(*in.Compiler)) + } + handler, err := in.Value.Compile(compilers) + if err != nil { + return nil, err + } + return func(resource any, bindings binding.Bindings) Feedback { + var out Feedback + if projected, err := handler(resource, bindings); err != nil { + out.Error = err + } else { + out.Value = projected + } + return out + }, nil +} + +func (c *compiler) compileRule( + path *field.Path, + compilers compilers.Compilers, + in v1alpha1.ValidatingRule, +) (func(any, binding.Bindings) []RuleResponse, error) { + if in.Compiler != nil { + compilers = compilers.WithDefaultCompiler(string(*in.Compiler)) + } + context, err := c.compileContext(path.Child("context"), compilers, in.Context...) + if err != nil { + return nil, err + } + identifier, err := c.compileIdentifier(path.Child("identifier"), compilers, in.Identifier) + if err != nil { + return nil, err + } + match, err := c.compileMatch(path.Child("match"), compilers, in.Match) + if err != nil { + return nil, err + } + exclude, err := c.compileMatch(path.Child("exclude"), compilers, in.Exclude) + if err != nil { + return nil, err + } + feedback, err := c.compileFeedbacks(path.Child("feedback"), compilers, in.Feedback...) + if err != nil { + return nil, err + } + // TODO: fix path + assert, err := c.compileAssert(nil, compilers, in.Assert) + if err != nil { + return nil, err + } + return func(resource any, bindings binding.Bindings) []RuleResponse { + // register context bindings + bindings = context(resource, bindings) + // process match clause + if match != nil { + if errs, err := match(resource, bindings); err != nil { + return []RuleResponse{{ + Rule: in, + Timestamp: time.Now(), + Identifier: identifier(resource, bindings), + Feedback: feedback(resource, bindings), + Error: err, + }} + } else if len(errs) != 0 { + // didn't match + return nil + } + } + // process exclude clause + if exclude != nil { + if errs, err := exclude(resource, bindings); err != nil { + return []RuleResponse{{ + Rule: in, + Timestamp: time.Now(), + Identifier: identifier(resource, bindings), + Feedback: feedback(resource, bindings), + Error: err, + }} + } else if len(errs) == 0 { + // matched + return nil + } + } + // evaluate assertions + violations, err := assert(resource, bindings) + if err != nil { + return []RuleResponse{{ + Rule: in, + Timestamp: time.Now(), + Identifier: identifier(resource, bindings), + Feedback: feedback(resource, bindings), + Error: err, + }} + } + return []RuleResponse{{ + Rule: in, + Timestamp: time.Now(), + Identifier: identifier(resource, bindings), + Feedback: feedback(resource, bindings), + Violations: violations, + }} + }, nil +} diff --git a/pkg/json-engine/engine.go b/pkg/json-engine/engine.go index 8df8ecc3..0774a75c 100644 --- a/pkg/json-engine/engine.go +++ b/pkg/json-engine/engine.go @@ -2,60 +2,13 @@ package jsonengine import ( "context" - "fmt" - "sync" "time" "github.com/jmespath-community/go-jmespath/pkg/binding" - jpbinding "github.com/jmespath-community/go-jmespath/pkg/binding" "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" "github.com/kyverno/kyverno-json/pkg/core/compilers" - "github.com/kyverno/kyverno-json/pkg/core/expression" "github.com/kyverno/kyverno-json/pkg/engine" "github.com/kyverno/kyverno-json/pkg/engine/builder" - "github.com/kyverno/kyverno-json/pkg/matching" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -type Request struct { - Resource any - Policies []*v1alpha1.ValidatingPolicy - Bindings map[string]any -} - -type Response struct { - Resource any - Policies []PolicyResponse -} - -type PolicyResponse struct { - Policy v1alpha1.ValidatingPolicy - Rules []RuleResponse -} - -type RuleResponse struct { - Rule v1alpha1.ValidatingRule - Timestamp time.Time - Identifier string - Feedback map[string]Feedback - Error error - Violations matching.Results -} - -type Feedback struct { - Error error - Value any -} - -// PolicyResult specifies state of a policy result -type PolicyResult string - -const ( - StatusPass PolicyResult = "pass" - StatusFail PolicyResult = "fail" - // StatusWarn PolicyResult = "warn" - StatusError PolicyResult = "error" - // StatusSkip PolicyResult = "skip" ) func New() engine.Engine[Request, Response] { @@ -63,151 +16,44 @@ func New() engine.Engine[Request, Response] { policy v1alpha1.ValidatingPolicy rule v1alpha1.ValidatingRule resource any - bindings jpbinding.Bindings + bindings binding.Bindings } type policyRequest struct { policy v1alpha1.ValidatingPolicy resource any - bindings jpbinding.Bindings + bindings binding.Bindings } - compiler := matching.NewCompiler(compilers.DefaultCompilers, 256) + compilers := compilers.DefaultCompilers + ruleCompiler := compiler{} ruleEngine := builder. Function(func(ctx context.Context, r ruleRequest) []RuleResponse { - bindings := r.bindings.Register("$rule", jpbinding.NewBinding(r.rule)) - defaultCompiler := expression.CompilerJP + compilers := compilers if r.policy.Spec.Compiler != nil { - defaultCompiler = string(*r.policy.Spec.Compiler) - } - if r.rule.Compiler != nil { - defaultCompiler = string(*r.rule.Compiler) - } - // TODO: this doesn't seem to be the right path - var path *field.Path - path = path.Child("context") - for _, entry := range r.rule.Context { - defaultCompiler := defaultCompiler - if entry.Compiler != nil { - defaultCompiler = string(*entry.Compiler) - } - bindings = func(variable v1alpha1.Any, bindings jpbinding.Bindings) jpbinding.Bindings { - return bindings.Register( - "$"+entry.Name, - binding.NewDelegate( - sync.OnceValues( - func() (any, error) { - handler, err := variable.Compile(compiler.CompileProjection, defaultCompiler) - if err != nil { - return nil, field.InternalError(path.Child("variable"), err) - } - projected, err := handler(r.resource, bindings) - if err != nil { - return nil, field.InternalError(path.Child("variable"), err) - } - return projected, nil - }, - ), - ), - ) - }(entry.Variable, bindings) - } - identifier := "" - if r.rule.Identifier != "" { - result, err := compilers.Execute(r.rule.Identifier, r.resource, bindings, compiler.Jp) - if err != nil { - identifier = fmt.Sprintf("(error: %s)", err) - } else { - identifier = fmt.Sprint(result) - } - } - if r.rule.Match != nil { - defaultCompiler := defaultCompiler - if r.rule.Match.Compiler != nil { - defaultCompiler = string(*r.rule.Match.Compiler) - } - errs, err := matching.Match(nil, r.rule.Match, r.resource, bindings, compiler, defaultCompiler) - if err != nil { - return []RuleResponse{{ - Rule: r.rule, - Timestamp: time.Now(), - Identifier: identifier, - Error: err, - }} - } - // didn't match - if len(errs) != 0 { - return nil - } - } - if r.rule.Exclude != nil { - defaultCompiler := defaultCompiler - if r.rule.Exclude.Compiler != nil { - defaultCompiler = string(*r.rule.Exclude.Compiler) - } - errs, err := matching.Match(nil, r.rule.Exclude, r.resource, bindings, compiler, defaultCompiler) - if err != nil { - return []RuleResponse{{ - Rule: r.rule, - Timestamp: time.Now(), - Identifier: identifier, - Error: err, - }} - } - // matched - if len(errs) == 0 { - return nil - } - } - var feedback map[string]Feedback - for _, f := range r.rule.Feedback { - entry := Feedback{} - if f.Value != nil { - defaultCompiler := defaultCompiler - if f.Compiler != nil { - defaultCompiler = string(*f.Compiler) - } - if handler, err := f.Value.Compile(compiler.CompileProjection, defaultCompiler); err != nil { - entry.Error = err - } else if projected, err := handler(r.resource, bindings); err != nil { - entry.Error = err - } else { - entry.Value = projected - } - } - if feedback == nil { - feedback = map[string]Feedback{} - } - feedback[f.Name] = entry + compilers = compilers.WithDefaultCompiler(string(*r.policy.Spec.Compiler)) } - violations, err := matching.Assert(nil, r.rule.Assert, r.resource, bindings, compiler, defaultCompiler) + compiled, err := ruleCompiler.compileRule(nil, compilers, r.rule) if err != nil { return []RuleResponse{{ - Rule: r.rule, - Timestamp: time.Now(), - Identifier: identifier, - Feedback: feedback, - Error: err, + Rule: r.rule, + Timestamp: time.Now(), + Error: err, }} } - return []RuleResponse{{ - Rule: r.rule, - Timestamp: time.Now(), - Identifier: identifier, - Feedback: feedback, - Violations: violations, - }} + bindings := r.bindings.Register("$rule", binding.NewBinding(r.rule)) + return compiled(r.resource, bindings) }) policyEngine := builder. Function(func(ctx context.Context, r policyRequest) PolicyResponse { response := PolicyResponse{ Policy: r.policy, } - bindings := r.bindings.Register("$policy", jpbinding.NewBinding(r.policy)) + bindings := r.bindings.Register("$policy", binding.NewBinding(r.policy)) for _, rule := range r.policy.Spec.Rules { response.Rules = append(response.Rules, ruleEngine.Run(ctx, ruleRequest{ policy: r.policy, rule: rule, resource: r.resource, - bindings: bindings.Register("$rule", jpbinding.NewBinding(rule)), + bindings: bindings.Register("$rule", binding.NewBinding(rule)), })...) } return response @@ -217,11 +63,11 @@ func New() engine.Engine[Request, Response] { response := Response{ Resource: r.Resource, } - bindings := jpbinding.NewBindings() + bindings := binding.NewBindings() for k, v := range r.Bindings { - bindings = bindings.Register("$"+k, jpbinding.NewBinding(v)) + bindings = bindings.Register("$"+k, binding.NewBinding(v)) } - bindings = bindings.Register("$payload", jpbinding.NewBinding(r.Resource)) + bindings = bindings.Register("$payload", binding.NewBinding(r.Resource)) for _, policy := range r.Policies { response.Policies = append(response.Policies, policyEngine.Run(ctx, policyRequest{ policy: *policy, diff --git a/pkg/json-engine/model.go b/pkg/json-engine/model.go new file mode 100644 index 00000000..bcb30f7c --- /dev/null +++ b/pkg/json-engine/model.go @@ -0,0 +1,78 @@ +package jsonengine + +import ( + "strings" + "time" + + "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +type Request struct { + Resource any + Policies []*v1alpha1.ValidatingPolicy + Bindings map[string]any +} + +type Response struct { + Resource any + Policies []PolicyResponse +} + +type PolicyResponse struct { + Policy v1alpha1.ValidatingPolicy + Rules []RuleResponse +} + +type RuleResponse struct { + Rule v1alpha1.ValidatingRule + Timestamp time.Time + Identifier string + Feedback map[string]Feedback + Error error + Violations Results +} + +//nolint:errname +type Result struct { + field.ErrorList + Message string +} + +func (r Result) Error() string { + var lines []string + if r.Message != "" { + lines = append(lines, "-> "+r.Message) + } + for _, err := range r.ErrorList { + lines = append(lines, " -> "+err.Error()) + } + return strings.Join(lines, "\n") +} + +//nolint:errname +type Results []Result + +func (r Results) Error() string { + var lines []string + for _, err := range r { + lines = append(lines, err.Error()) + } + return strings.Join(lines, "\n") +} + +type Feedback struct { + Error error + Value any +} + +// PolicyResult specifies state of a policy result +type PolicyResult string + +const ( + StatusPass PolicyResult = "pass" + StatusFail PolicyResult = "fail" + // StatusWarn PolicyResult = "warn" + StatusError PolicyResult = "error" + // StatusSkip PolicyResult = "skip" +) diff --git a/pkg/matching/compiler.go b/pkg/matching/compiler.go deleted file mode 100644 index 088ae2b4..00000000 --- a/pkg/matching/compiler.go +++ /dev/null @@ -1,52 +0,0 @@ -package matching - -import ( - "sync" - - "github.com/cespare/xxhash/v2" - "github.com/elastic/go-freelru" - "github.com/kyverno/kyverno-json/pkg/core/assertion" - "github.com/kyverno/kyverno-json/pkg/core/compilers" - "github.com/kyverno/kyverno-json/pkg/core/projection" -) - -type _compilers = compilers.Compilers - -type Compiler struct { - _compilers - *freelru.SyncedLRU[string, func() (assertion.Assertion, error)] -} - -func hashStringXXHASH(s string) uint32 { - sum := xxhash.Sum64String(s) - return uint32(sum) //nolint:gosec -} - -func NewCompiler(compiler compilers.Compilers, cacheSize uint32) Compiler { - out := Compiler{ - _compilers: compiler, - } - if cache, err := freelru.NewSynced[string, func() (assertion.Assertion, error)](cacheSize, hashStringXXHASH); err == nil { - out.SyncedLRU = cache - } - return out -} - -func (c Compiler) CompileAssertion(hash string, value any, defaultCompiler string) (assertion.Assertion, error) { - if c.SyncedLRU == nil || hash == "" { - return assertion.Parse(value, c._compilers.WithDefaultCompiler(defaultCompiler)) - } - entry, _ := c.SyncedLRU.Get(hash) - if entry == nil { - entry = sync.OnceValues(func() (assertion.Assertion, error) { - return assertion.Parse(value, c._compilers.WithDefaultCompiler(defaultCompiler)) - }) - c.SyncedLRU.Add(hash, entry) - } - return entry() -} - -func (c Compiler) CompileProjection(hash string, value any, defaultCompiler string) (projection.ScalarHandler, error) { - // TODO: cache - return projection.ParseScalar(value, c._compilers.WithDefaultCompiler(defaultCompiler)) -} diff --git a/pkg/matching/match.go b/pkg/matching/match.go deleted file mode 100644 index 9a91c5c9..00000000 --- a/pkg/matching/match.go +++ /dev/null @@ -1,161 +0,0 @@ -package matching - -import ( - "strings" - - "github.com/jmespath-community/go-jmespath/pkg/binding" - "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -//nolint:errname -type Result struct { - field.ErrorList - Message string -} - -func (r Result) Error() string { - var lines []string - if r.Message != "" { - lines = append(lines, "-> "+r.Message) - } - for _, err := range r.ErrorList { - lines = append(lines, " -> "+err.Error()) - } - return strings.Join(lines, "\n") -} - -//nolint:errname -type Results []Result - -func (r Results) Error() string { - var lines []string - for _, err := range r { - lines = append(lines, err.Error()) - } - return strings.Join(lines, "\n") -} - -func Assert(path *field.Path, in v1alpha1.Assert, actual any, bindings binding.Bindings, compiler Compiler, defaultCompiler string) ([]Result, error) { - if in.Compiler != nil { - defaultCompiler = string(*in.Compiler) - } - if len(in.Any) == 0 && len(in.All) == 0 { - return nil, field.Invalid(path, in, "an empty assert is not valid") - } else { - if len(in.Any) != 0 { - var fails []Result - path := path.Child("any") - for i, assertion := range in.Any { - defaultCompiler := defaultCompiler - if assertion.Compiler != nil { - defaultCompiler = string(*assertion.Compiler) - } - checkFails, err := assert(path.Index(i).Child("check"), assertion.Check, actual, bindings, compiler, defaultCompiler) - if err != nil { - return fails, err - } - if len(checkFails) == 0 { - fails = nil - break - } - fail := Result{ - ErrorList: checkFails, - } - if assertion.Message != nil { - fail.Message = assertion.Message.Format(actual, bindings, compiler.Jp.Options()...) - } - fails = append(fails, fail) - } - if fails != nil { - return fails, nil - } - } - if len(in.All) != 0 { - var fails []Result - path := path.Child("all") - for i, assertion := range in.All { - defaultCompiler := defaultCompiler - if assertion.Compiler != nil { - defaultCompiler = string(*assertion.Compiler) - } - checkFails, err := assert(path.Index(i).Child("check"), assertion.Check, actual, bindings, compiler, defaultCompiler) - if err != nil { - return fails, err - } - if len(checkFails) > 0 { - fail := Result{ - ErrorList: checkFails, - } - if assertion.Message != nil { - fail.Message = assertion.Message.Format(actual, bindings, compiler.Jp.Options()...) - } - fails = append(fails, fail) - } - } - return fails, nil - } - return nil, nil - } -} - -func Match(path *field.Path, in *v1alpha1.Match, actual any, bindings binding.Bindings, compiler Compiler, defaultCompiler string) (field.ErrorList, error) { - if in.Compiler != nil { - defaultCompiler = string(*in.Compiler) - } - if in == nil || (len(in.Any) == 0 && len(in.All) == 0) { - return nil, field.Invalid(path, in, "an empty match is not valid") - } else { - var errs field.ErrorList - if len(in.Any) != 0 { - _errs, err := matchAny(path.Child("any"), in.Any, actual, bindings, compiler, defaultCompiler) - if err != nil { - return errs, err - } - errs = append(errs, _errs...) - } - if len(in.All) != 0 { - _errs, err := matchAll(path.Child("all"), in.All, actual, bindings, compiler, defaultCompiler) - if err != nil { - return errs, err - } - errs = append(errs, _errs...) - } - return errs, nil - } -} - -func matchAny(path *field.Path, in []v1alpha1.AssertionTree, value any, bindings binding.Bindings, compiler Compiler, defaultCompiler string) (field.ErrorList, error) { - var errs field.ErrorList - for i, assertion := range in { - _errs, err := assert(path.Index(i), assertion, value, bindings, compiler, defaultCompiler) - if err != nil { - return errs, err - } - if len(_errs) == 0 { - return nil, nil - } - errs = append(errs, _errs...) - } - return errs, nil -} - -func matchAll(path *field.Path, in []v1alpha1.AssertionTree, value any, bindings binding.Bindings, compiler Compiler, defaultCompiler string) (field.ErrorList, error) { - var errs field.ErrorList - for i, assertion := range in { - _errs, err := assert(path.Index(i), assertion, value, bindings, compiler, defaultCompiler) - if err != nil { - return errs, err - } - errs = append(errs, _errs...) - } - return errs, nil -} - -func assert(path *field.Path, assertion v1alpha1.AssertionTree, value any, bindings binding.Bindings, compiler Compiler, defaultCompiler string) (field.ErrorList, error) { - check, err := assertion.Compile(compiler.CompileAssertion, defaultCompiler) - if err != nil { - return nil, err - } - return check.Assert(path, value, bindings) -}