refactor: policy compilation (#516)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
kyverno · Sep 24, 2024 · 4cb9d0f · 4cb9d0f
1 parent 9d7931b
commit 4cb9d0f
Show file tree

Hide file tree

Showing 12 changed files with 613 additions and 514 deletions.
diff --git a/go.mod b/go.mod
@@ -5,8 +5,6 @@ go 1.22.2
 require (
 	github.com/aquilax/truncate v1.0.0
 	github.com/blang/semver/v4 v4.0.0
-	github.com/cespare/xxhash/v2 v2.3.0
-	github.com/elastic/go-freelru v0.13.0
 	github.com/gin-contrib/cors v1.7.2
 	github.com/gin-gonic/gin v1.10.0
 	github.com/google/cel-go v0.20.1
@@ -38,6 +36,7 @@ require (
 	github.com/bytedance/sonic v1.12.2 // indirect
 	github.com/bytedance/sonic/loader v0.2.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect

diff --git a/go.sum b/go.sum
@@ -47,8 +47,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 h1:S6Dco8FtAhEI/qkg/00H6RdEGC+MCy5GPiQ+xweNRFE=
 github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc=
-github.com/elastic/go-freelru v0.13.0 h1:TKKY6yCfNNNky7Pj9xZAOEpBcdNgZJfihEftOb55omg=
-github.com/elastic/go-freelru v0.13.0/go.mod h1:bSdWT4M0lW79K8QbX6XY2heQYSCqD7THoYf82pT/H3I=
 github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
 github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

diff --git a/pkg/apis/policy/v1alpha1/any.go b/pkg/apis/policy/v1alpha1/any.go
@@ -1,8 +1,8 @@
 package v1alpha1
 
 import (
+	"github.com/kyverno/kyverno-json/pkg/core/compilers"
 	"github.com/kyverno/kyverno-json/pkg/core/projection"
-	hashutils "github.com/kyverno/kyverno-json/pkg/utils/hash"
 	"k8s.io/apimachinery/pkg/util/json"
 )
 
@@ -12,18 +12,16 @@ import (
 // +kubebuilder:validation:Type:=""
 type Any struct {
 	_value any
-	_hash  string
 }
 
 func NewAny(value any) Any {
 	return Any{
 		_value: value,
-		_hash:  hashutils.Hash(value),
 	}
 }
 
-func (t *Any) Compile(compiler func(string, any, string) (projection.ScalarHandler, error), defaultCompiler string) (projection.ScalarHandler, error) {
-	return compiler(t._hash, t._value, defaultCompiler)
+func (t *Any) Compile(compilers compilers.Compilers) (projection.ScalarHandler, error) {
+	return projection.ParseScalar(t._value, compilers)
 }
 
 func (a *Any) MarshalJSON() ([]byte, error) {
@@ -37,13 +35,11 @@ func (a *Any) UnmarshalJSON(data []byte) error {
 		return err
 	}
 	a._value = v
-	a._hash = hashutils.Hash(a._value)
 	return nil
 }
 
 func (in *Any) DeepCopyInto(out *Any) {
 	out._value = deepCopy(in._value)
-	out._hash = in._hash
 }
 
 func (in *Any) DeepCopy() *Any {

diff --git a/pkg/apis/policy/v1alpha1/assertion_tree.go b/pkg/apis/policy/v1alpha1/assertion_tree.go
@@ -2,7 +2,7 @@ package v1alpha1
 
 import (
 	"github.com/kyverno/kyverno-json/pkg/core/assertion"
-	hashutils "github.com/kyverno/kyverno-json/pkg/utils/hash"
+	"github.com/kyverno/kyverno-json/pkg/core/compilers"
 	"k8s.io/apimachinery/pkg/util/json"
 )
 
@@ -12,18 +12,16 @@ import (
 // AssertionTree represents an assertion tree.
 type AssertionTree struct {
 	_tree any
-	_hash string
 }
 
 func NewAssertionTree(value any) AssertionTree {
 	return AssertionTree{
 		_tree: value,
-		_hash: hashutils.Hash(value),
 	}
 }
 
-func (t *AssertionTree) Compile(compiler func(string, any, string) (assertion.Assertion, error), defaultCompiler string) (assertion.Assertion, error) {
-	return compiler(t._hash, t._tree, defaultCompiler)
+func (t *AssertionTree) Compile(compilers compilers.Compilers) (assertion.Assertion, error) {
+	return assertion.Parse(t._tree, compilers)
 }
 
 func (a *AssertionTree) MarshalJSON() ([]byte, error) {
@@ -37,11 +35,9 @@ func (a *AssertionTree) UnmarshalJSON(data []byte) error {
 		return err
 	}
 	a._tree = v
-	a._hash = hashutils.Hash(a._tree)
 	return nil
 }
 
 func (in *AssertionTree) DeepCopyInto(out *AssertionTree) {
 	out._tree = deepCopy(in._tree)
-	out._hash = in._hash
 }
diff --git a/pkg/commands/scan/command_test.go b/pkg/commands/scan/command_test.go
@@ -19,122 +19,122 @@ func Test_Execute(t *testing.T) {
 		wantErr       bool
 		out           string
 	}{{
-		// 	name:     "foo-bar",
-		// 	payload:  "../../../test/commands/scan/foo-bar/payload.yaml",
-		// 	policies: []string{"../../../test/commands/scan/foo-bar/policy.yaml"},
-		// 	out:      "../../../test/commands/scan/foo-bar/out.txt",
-		// 	wantErr:  false,
-		// }, {
+		name:     "foo-bar",
+		payload:  "../../../test/commands/scan/foo-bar/payload.yaml",
+		policies: []string{"../../../test/commands/scan/foo-bar/policy.yaml"},
+		out:      "../../../test/commands/scan/foo-bar/out.txt",
+		wantErr:  false,
+	}, {
 		name:     "cel",
 		payload:  "../../../test/commands/scan/cel/payload.yaml",
 		policies: []string{"../../../test/commands/scan/cel/policy.yaml"},
 		out:      "../../../test/commands/scan/cel/out.txt",
 		wantErr:  false,
-		// }, {
-		// 	name:     "wildcard",
-		// 	payload:  "../../../test/commands/scan/wildcard/payload.json",
-		// 	policies: []string{"../../../test/commands/scan/wildcard/policy.yaml"},
-		// 	out:      "../../../test/commands/scan/wildcard/out.txt",
-		// 	wantErr:  false,
-		// }, {
-		// 	name:     "bindings",
-		// 	bindings: "../../../test/commands/scan/bindings/bindings.yaml",
-		// 	payload:  "../../../test/commands/scan/bindings/payload.yaml",
-		// 	policies: []string{"../../../test/commands/scan/bindings/policy.yaml"},
-		// 	out:      "../../../test/commands/scan/bindings/out.txt",
-		// 	wantErr:  false,
-		// }, {
-		// 	name:     "pod-no-latest",
-		// 	payload:  "../../../test/commands/scan/pod-no-latest/payload.yaml",
-		// 	policies: []string{"../../../test/commands/scan/pod-no-latest/policy.yaml"},
-		// 	out:      "../../../test/commands/scan/pod-no-latest/out.txt",
-		// 	wantErr:  false,
-		// }, {
-		// 	name:     "pod-all-latest",
-		// 	payload:  "../../../test/commands/scan/pod-all-latest/payload.yaml",
-		// 	policies: []string{"../../../test/commands/scan/pod-all-latest/policy.yaml"},
-		// 	out:      "../../../test/commands/scan/pod-all-latest/out.txt",
-		// 	wantErr:  false,
-		// }, {
-		// 	name:     "scripted",
-		// 	payload:  "../../../test/commands/scan/scripted/payload.yaml",
-		// 	policies: []string{"../../../test/commands/scan/scripted/policy.yaml"},
-		// 	out:      "../../../test/commands/scan/scripted/out.txt",
-		// 	wantErr:  false,
-		// }, {
-		// 	name:          "payload-yaml",
-		// 	payload:       "../../../test/commands/scan/payload-yaml/payload.yaml",
-		// 	preprocessors: []string{"planned_values.root_module.resources"},
-		// 	policies:      []string{"../../../test/commands/scan/payload-yaml/policy.yaml"},
-		// 	out:           "../../../test/commands/scan/payload-yaml/out.txt",
-		// 	wantErr:       false,
-		// }, {
-		// 	name:          "tf-plan",
-		// 	payload:       "../../../test/commands/scan/tf-plan/payload.json",
-		// 	preprocessors: []string{"planned_values.root_module.resources"},
-		// 	policies:      []string{"../../../test/commands/scan/tf-plan/policy.yaml"},
-		// 	out:           "../../../test/commands/scan/tf-plan/out.txt",
-		// 	wantErr:       false,
-		// }, {
-		// 	name:     "escaped",
-		// 	payload:  "../../../test/commands/scan/escaped/payload.yaml",
-		// 	policies: []string{"../../../test/commands/scan/escaped/policy.yaml"},
-		// 	out:      "../../../test/commands/scan/escaped/out.txt",
-		// 	wantErr:  false,
-		// }, {
-		// 	name:     "dockerfile",
-		// 	payload:  "../../../test/commands/scan/dockerfile/payload.json",
-		// 	policies: []string{"../../../test/commands/scan/dockerfile/policy.yaml"},
-		// 	out:      "../../../test/commands/scan/dockerfile/out.txt",
-		// 	wantErr:  false,
-		// }, {
-		// 	name:     "tf-s3",
-		// 	payload:  "../../../test/commands/scan/tf-s3/payload.json",
-		// 	policies: []string{"../../../test/commands/scan/tf-s3/policy.yaml"},
-		// 	out:      "../../../test/commands/scan/tf-s3/out.txt",
-		// 	wantErr:  false,
-		// }, {
-		// 	name:          "tf-ec2",
-		// 	payload:       "../../../test/commands/scan/tf-ec2/payload.json",
-		// 	preprocessors: []string{"planned_values.root_module.resources"},
-		// 	policies:      []string{"../../../test/commands/scan/tf-ec2/policy.yaml"},
-		// 	out:           "../../../test/commands/scan/tf-ec2/out.txt",
-		// 	wantErr:       false,
-		// }, {
-		// 	name:          "tf-ecs-cluster-1",
-		// 	payload:       "../../../test/commands/scan/tf-ecs-cluster/payload.json",
-		// 	preprocessors: []string{"planned_values.root_module.resources"},
-		// 	policies:      []string{"../../../test/commands/scan/tf-ecs-cluster/01-policy.yaml"},
-		// 	out:           "../../../test/commands/scan/tf-ecs-cluster/01-out.txt",
-		// 	wantErr:       false,
-		// }, {
-		// 	name:          "tf-ecs-cluster-2",
-		// 	payload:       "../../../test/commands/scan/tf-ecs-cluster/payload.json",
-		// 	preprocessors: []string{"planned_values.root_module.resources"},
-		// 	policies:      []string{"../../../test/commands/scan/tf-ecs-cluster/02-policy.yaml"},
-		// 	out:           "../../../test/commands/scan/tf-ecs-cluster/02-out.txt",
-		// 	wantErr:       false,
-		// }, {
-		// 	name:          "tf-ecs-service-1",
-		// 	payload:       "../../../test/commands/scan/tf-ecs-service/payload.json",
-		// 	preprocessors: []string{"planned_values.root_module.resources"},
-		// 	policies:      []string{"../../../test/commands/scan/tf-ecs-service/01-policy.yaml"},
-		// 	out:           "../../../test/commands/scan/tf-ecs-service/01-out.txt",
-		// 	wantErr:       false,
-		// }, {
-		// 	name:          "tf-ecs-service-2",
-		// 	payload:       "../../../test/commands/scan/tf-ecs-service/payload.json",
-		// 	preprocessors: []string{"planned_values.root_module.resources"},
-		// 	policies:      []string{"../../../test/commands/scan/tf-ecs-service/02-policy.yaml"},
-		// 	out:           "../../../test/commands/scan/tf-ecs-service/02-out.txt",
-		// 	wantErr:       false,
-		// }, {
-		// 	name:          "tf-ecs-task-definition",
-		// 	payload:       "../../../test/commands/scan/tf-ecs-task-definition/payload.json",
-		// 	preprocessors: []string{"planned_values.root_module.resources"},
-		// 	policies:      []string{"../../../test/commands/scan/tf-ecs-task-definition/policy.yaml"},
-		// 	out:           "../../../test/commands/scan/tf-ecs-task-definition/out.txt",
-		// 	wantErr:       false,
+	}, {
+		name:     "wildcard",
+		payload:  "../../../test/commands/scan/wildcard/payload.json",
+		policies: []string{"../../../test/commands/scan/wildcard/policy.yaml"},
+		out:      "../../../test/commands/scan/wildcard/out.txt",
+		wantErr:  false,
+	}, {
+		name:     "bindings",
+		bindings: "../../../test/commands/scan/bindings/bindings.yaml",
+		payload:  "../../../test/commands/scan/bindings/payload.yaml",
+		policies: []string{"../../../test/commands/scan/bindings/policy.yaml"},
+		out:      "../../../test/commands/scan/bindings/out.txt",
+		wantErr:  false,
+	}, {
+		name:     "pod-no-latest",
+		payload:  "../../../test/commands/scan/pod-no-latest/payload.yaml",
+		policies: []string{"../../../test/commands/scan/pod-no-latest/policy.yaml"},
+		out:      "../../../test/commands/scan/pod-no-latest/out.txt",
+		wantErr:  false,
+	}, {
+		name:     "pod-all-latest",
+		payload:  "../../../test/commands/scan/pod-all-latest/payload.yaml",
+		policies: []string{"../../../test/commands/scan/pod-all-latest/policy.yaml"},
+		out:      "../../../test/commands/scan/pod-all-latest/out.txt",
+		wantErr:  false,
+	}, {
+		name:     "scripted",
+		payload:  "../../../test/commands/scan/scripted/payload.yaml",
+		policies: []string{"../../../test/commands/scan/scripted/policy.yaml"},
+		out:      "../../../test/commands/scan/scripted/out.txt",
+		wantErr:  false,
+	}, {
+		name:          "payload-yaml",
+		payload:       "../../../test/commands/scan/payload-yaml/payload.yaml",
+		preprocessors: []string{"planned_values.root_module.resources"},
+		policies:      []string{"../../../test/commands/scan/payload-yaml/policy.yaml"},
+		out:           "../../../test/commands/scan/payload-yaml/out.txt",
+		wantErr:       false,
+	}, {
+		name:          "tf-plan",
+		payload:       "../../../test/commands/scan/tf-plan/payload.json",
+		preprocessors: []string{"planned_values.root_module.resources"},
+		policies:      []string{"../../../test/commands/scan/tf-plan/policy.yaml"},
+		out:           "../../../test/commands/scan/tf-plan/out.txt",
+		wantErr:       false,
+	}, {
+		name:     "escaped",
+		payload:  "../../../test/commands/scan/escaped/payload.yaml",
+		policies: []string{"../../../test/commands/scan/escaped/policy.yaml"},
+		out:      "../../../test/commands/scan/escaped/out.txt",
+		wantErr:  false,
+	}, {
+		name:     "dockerfile",
+		payload:  "../../../test/commands/scan/dockerfile/payload.json",
+		policies: []string{"../../../test/commands/scan/dockerfile/policy.yaml"},
+		out:      "../../../test/commands/scan/dockerfile/out.txt",
+		wantErr:  false,
+	}, {
+		name:     "tf-s3",
+		payload:  "../../../test/commands/scan/tf-s3/payload.json",
+		policies: []string{"../../../test/commands/scan/tf-s3/policy.yaml"},
+		out:      "../../../test/commands/scan/tf-s3/out.txt",
+		wantErr:  false,
+	}, {
+		name:          "tf-ec2",
+		payload:       "../../../test/commands/scan/tf-ec2/payload.json",
+		preprocessors: []string{"planned_values.root_module.resources"},
+		policies:      []string{"../../../test/commands/scan/tf-ec2/policy.yaml"},
+		out:           "../../../test/commands/scan/tf-ec2/out.txt",
+		wantErr:       false,
+	}, {
+		name:          "tf-ecs-cluster-1",
+		payload:       "../../../test/commands/scan/tf-ecs-cluster/payload.json",
+		preprocessors: []string{"planned_values.root_module.resources"},
+		policies:      []string{"../../../test/commands/scan/tf-ecs-cluster/01-policy.yaml"},
+		out:           "../../../test/commands/scan/tf-ecs-cluster/01-out.txt",
+		wantErr:       false,
+	}, {
+		name:          "tf-ecs-cluster-2",
+		payload:       "../../../test/commands/scan/tf-ecs-cluster/payload.json",
+		preprocessors: []string{"planned_values.root_module.resources"},
+		policies:      []string{"../../../test/commands/scan/tf-ecs-cluster/02-policy.yaml"},
+		out:           "../../../test/commands/scan/tf-ecs-cluster/02-out.txt",
+		wantErr:       false,
+	}, {
+		name:          "tf-ecs-service-1",
+		payload:       "../../../test/commands/scan/tf-ecs-service/payload.json",
+		preprocessors: []string{"planned_values.root_module.resources"},
+		policies:      []string{"../../../test/commands/scan/tf-ecs-service/01-policy.yaml"},
+		out:           "../../../test/commands/scan/tf-ecs-service/01-out.txt",
+		wantErr:       false,
+	}, {
+		name:          "tf-ecs-service-2",
+		payload:       "../../../test/commands/scan/tf-ecs-service/payload.json",
+		preprocessors: []string{"planned_values.root_module.resources"},
+		policies:      []string{"../../../test/commands/scan/tf-ecs-service/02-policy.yaml"},
+		out:           "../../../test/commands/scan/tf-ecs-service/02-out.txt",
+		wantErr:       false,
+	}, {
+		name:          "tf-ecs-task-definition",
+		payload:       "../../../test/commands/scan/tf-ecs-task-definition/payload.json",
+		preprocessors: []string{"planned_values.root_module.resources"},
+		policies:      []string{"../../../test/commands/scan/tf-ecs-task-definition/policy.yaml"},
+		out:           "../../../test/commands/scan/tf-ecs-task-definition/out.txt",
+		wantErr:       false,
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

diff --git a/pkg/core/compilers/compilers.go b/pkg/core/compilers/compilers.go
@@ -6,6 +6,11 @@ import (
 	"github.com/kyverno/kyverno-json/pkg/core/expression"
 )
 
+const (
+	CompilerCEL = expression.CompilerCEL
+	CompilerJP  = expression.CompilerJP
+)
+
 var DefaultCompilers = Compilers{
 	Jp:  jp.NewCompiler(),
 	Cel: cel.NewCompiler(),

diff --git a/pkg/core/expression/expression.go b/pkg/core/expression/expression.go
@@ -5,8 +5,8 @@ import (
 )
 
 const (
-	CompilerJP      = "jp"
 	CompilerCEL     = "cel"
+	CompilerJP      = "jp"
 	CompilerDefault = "default"
 )