From a7d5579a7919c64dca062f9624a15a295e5e30cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Sat, 21 Sep 2024 13:29:31 +0200 Subject: [PATCH] feat: improve compilers package (#507) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché --- pkg/commands/jp/query/command.go | 5 ++--- pkg/commands/scan/options.go | 5 ++--- pkg/core/assertion/assertion.go | 9 ++++----- pkg/core/assertion/assertion_test.go | 4 ++-- pkg/core/compilers/cel/cel.go | 7 ++++--- .../compiler.go => compilers/compilers.go} | 8 ++++---- .../templating.go => compilers/execute.go} | 5 ++--- pkg/core/compilers/jp/jp.go | 7 ++++--- pkg/core/compilers/program.go | 2 +- pkg/core/projection/projection.go | 3 +-- pkg/core/projection/projection_test.go | 4 ++-- pkg/json-engine/engine.go | 8 ++++---- pkg/matching/compiler.go | 12 ++++++------ pkg/server/playground/handler.go | 4 ++-- pkg/server/scan/handler.go | 4 ++-- 15 files changed, 42 insertions(+), 45 deletions(-) rename pkg/core/{templating/compiler.go => compilers/compilers.go} (88%) rename pkg/core/{templating/templating.go => compilers/execute.go} (67%) diff --git a/pkg/commands/jp/query/command.go b/pkg/commands/jp/query/command.go index da79b58b..73aa2ccc 100644 --- a/pkg/commands/jp/query/command.go +++ b/pkg/commands/jp/query/command.go @@ -10,7 +10,7 @@ import ( "github.com/jmespath-community/go-jmespath/pkg/parsing" "github.com/kyverno/kyverno-json/pkg/command" - "github.com/kyverno/kyverno-json/pkg/core/templating" + "github.com/kyverno/kyverno-json/pkg/core/compilers" "github.com/spf13/cobra" "sigs.k8s.io/yaml" ) @@ -155,8 +155,7 @@ func loadInput(cmd *cobra.Command, file string) (any, error) { } func evaluate(input any, query string) (any, error) { - compiler := templating.DefaultCompiler - result, err := templating.Execute(query, input, nil, compiler.Jp) + result, err := compilers.Execute(query, input, nil, compilers.DefaultCompiler.Jp) if err != nil { if syntaxError, ok := err.(parsing.SyntaxError); ok { return nil, fmt.Errorf("%s\n%s", syntaxError, syntaxError.HighlightLocation()) diff --git a/pkg/commands/scan/options.go b/pkg/commands/scan/options.go index c0277b73..15049be4 100644 --- a/pkg/commands/scan/options.go +++ b/pkg/commands/scan/options.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" - "github.com/kyverno/kyverno-json/pkg/core/templating" + "github.com/kyverno/kyverno-json/pkg/core/compilers" jsonengine "github.com/kyverno/kyverno-json/pkg/json-engine" "github.com/kyverno/kyverno-json/pkg/payload" "github.com/kyverno/kyverno-json/pkg/policy" @@ -76,9 +76,8 @@ func (c *options) run(cmd *cobra.Command, _ []string) error { return errors.New("payload is `null`") } out.println("Pre processing ...") - compiler := templating.DefaultCompiler for _, preprocessor := range c.preprocessors { - result, err := templating.Execute(preprocessor, payload, nil, compiler.Jp) + result, err := compilers.Execute(preprocessor, payload, nil, compilers.DefaultCompiler.Jp) if err != nil { return err } diff --git a/pkg/core/assertion/assertion.go b/pkg/core/assertion/assertion.go index 4fd1b347..c8b61eb7 100644 --- a/pkg/core/assertion/assertion.go +++ b/pkg/core/assertion/assertion.go @@ -11,7 +11,6 @@ import ( "github.com/kyverno/kyverno-json/pkg/core/expression" "github.com/kyverno/kyverno-json/pkg/core/matching" "github.com/kyverno/kyverno-json/pkg/core/projection" - "github.com/kyverno/kyverno-json/pkg/core/templating" reflectutils "github.com/kyverno/kyverno-json/pkg/utils/reflect" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -20,7 +19,7 @@ type Assertion interface { Assert(*field.Path, any, binding.Bindings) (field.ErrorList, error) } -func Parse(assertion any, compiler templating.Compiler) (node, error) { +func Parse(assertion any, compiler compilers.Compilers) (node, error) { switch reflectutils.GetKind(assertion) { case reflect.Slice: return parseSlice(assertion, compiler) @@ -41,7 +40,7 @@ func (n node) Assert(path *field.Path, value any, bindings binding.Bindings) (fi // parseSlice is the assertion represented by a slice. // it first compares the length of the analysed resource with the length of the descendants. // if lengths match all descendants are evaluated with their corresponding items. -func parseSlice(assertion any, compiler templating.Compiler) (node, error) { +func parseSlice(assertion any, compiler compilers.Compilers) (node, error) { var assertions []node valueOf := reflect.ValueOf(assertion) for i := 0; i < valueOf.Len(); i++ { @@ -77,7 +76,7 @@ func parseSlice(assertion any, compiler templating.Compiler) (node, error) { // parseMap is the assertion represented by a map. // it is responsible for projecting the analysed resource and passing the result to the descendant -func parseMap(assertion any, compiler templating.Compiler) (node, error) { +func parseMap(assertion any, compiler compilers.Compilers) (node, error) { assertions := map[any]struct { projection.Projection node @@ -162,7 +161,7 @@ func parseMap(assertion any, compiler templating.Compiler) (node, error) { // parseScalar is the assertion represented by a leaf. // it receives a value and compares it with an expected value. // the expected value can be the result of an expression. -func parseScalar(assertion any, compiler templating.Compiler) (node, error) { +func parseScalar(assertion any, compiler compilers.Compilers) (node, error) { var project func(value any, bindings binding.Bindings) (any, error) switch typed := assertion.(type) { case string: diff --git a/pkg/core/assertion/assertion_test.go b/pkg/core/assertion/assertion_test.go index a796d3bc..520fec37 100644 --- a/pkg/core/assertion/assertion_test.go +++ b/pkg/core/assertion/assertion_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/jmespath-community/go-jmespath/pkg/binding" - "github.com/kyverno/kyverno-json/pkg/core/templating" + "github.com/kyverno/kyverno-json/pkg/core/compilers" tassert "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -48,7 +48,7 @@ func TestAssert(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler := templating.DefaultCompiler + compiler := compilers.DefaultCompiler parsed, err := Parse(tt.assertion, compiler) tassert.NoError(t, err) got, err := parsed.Assert(nil, tt.value, tt.bindings) diff --git a/pkg/core/compilers/cel/cel.go b/pkg/core/compilers/cel/cel.go index 5468f508..e1967564 100644 --- a/pkg/core/compilers/cel/cel.go +++ b/pkg/core/compilers/cel/cel.go @@ -3,11 +3,12 @@ package cel import ( "github.com/google/cel-go/cel" "github.com/jmespath-community/go-jmespath/pkg/binding" - "github.com/kyverno/kyverno-json/pkg/core/compilers" ) +type Program = func(any, binding.Bindings) (any, error) + type Compiler interface { - Compile(string) (compilers.Program, error) + Compile(string) (Program, error) } type compiler struct{} @@ -16,7 +17,7 @@ func NewCompiler() *compiler { return &compiler{} } -func (c *compiler) Compile(statement string) (compilers.Program, error) { +func (c *compiler) Compile(statement string) (Program, error) { env, err := DefaultEnv() if err != nil { return nil, err diff --git a/pkg/core/templating/compiler.go b/pkg/core/compilers/compilers.go similarity index 88% rename from pkg/core/templating/compiler.go rename to pkg/core/compilers/compilers.go index 2538091b..a0e2b3ef 100644 --- a/pkg/core/templating/compiler.go +++ b/pkg/core/compilers/compilers.go @@ -1,4 +1,4 @@ -package templating +package compilers import ( "sync" @@ -10,17 +10,17 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) -var DefaultCompiler = Compiler{ +var DefaultCompiler = Compilers{ Jp: jp.NewCompiler(), Cel: cel.NewCompiler(), } -type Compiler struct { +type Compilers struct { Jp jp.Compiler Cel cel.Compiler } -func (c Compiler) NewBinding(path *field.Path, value any, bindings binding.Bindings, template any) binding.Binding { +func (c Compilers) NewBinding(path *field.Path, value any, bindings binding.Bindings, template any) binding.Binding { return binding.NewDelegate( sync.OnceValues( func() (any, error) { diff --git a/pkg/core/templating/templating.go b/pkg/core/compilers/execute.go similarity index 67% rename from pkg/core/templating/templating.go rename to pkg/core/compilers/execute.go index 1d54aef6..29d51a9d 100644 --- a/pkg/core/templating/templating.go +++ b/pkg/core/compilers/execute.go @@ -1,11 +1,10 @@ -package templating +package compilers import ( "github.com/jmespath-community/go-jmespath/pkg/binding" - "github.com/kyverno/kyverno-json/pkg/core/compilers" ) -func Execute(statement string, value any, bindings binding.Bindings, compiler compilers.Compiler) (any, error) { +func Execute(statement string, value any, bindings binding.Bindings, compiler Compiler) (any, error) { program, err := compiler.Compile(statement) if err != nil { return nil, err diff --git a/pkg/core/compilers/jp/jp.go b/pkg/core/compilers/jp/jp.go index 9e499f12..0dfa7a8a 100644 --- a/pkg/core/compilers/jp/jp.go +++ b/pkg/core/compilers/jp/jp.go @@ -6,11 +6,12 @@ import ( "github.com/jmespath-community/go-jmespath/pkg/binding" "github.com/jmespath-community/go-jmespath/pkg/interpreter" "github.com/jmespath-community/go-jmespath/pkg/parsing" - "github.com/kyverno/kyverno-json/pkg/core/compilers" ) +type Program = func(any, binding.Bindings) (any, error) + type Compiler interface { - Compile(string) (compilers.Program, error) + Compile(string) (Program, error) Options() []Option } @@ -32,7 +33,7 @@ func (c *compiler) Options() []Option { return c.options } -func (c *compiler) Compile(statement string) (compilers.Program, error) { +func (c *compiler) Compile(statement string) (Program, error) { parser := parsing.NewParser() compiled, err := parser.Parse(statement) if err != nil { diff --git a/pkg/core/compilers/program.go b/pkg/core/compilers/program.go index 6772968a..9f3156af 100644 --- a/pkg/core/compilers/program.go +++ b/pkg/core/compilers/program.go @@ -4,4 +4,4 @@ import ( "github.com/jmespath-community/go-jmespath/pkg/binding" ) -type Program func(any, binding.Bindings) (any, error) +type Program = func(any, binding.Bindings) (any, error) diff --git a/pkg/core/projection/projection.go b/pkg/core/projection/projection.go index 40e73b1f..a493ed40 100644 --- a/pkg/core/projection/projection.go +++ b/pkg/core/projection/projection.go @@ -8,7 +8,6 @@ import ( "github.com/jmespath-community/go-jmespath/pkg/binding" "github.com/kyverno/kyverno-json/pkg/core/compilers" "github.com/kyverno/kyverno-json/pkg/core/expression" - "github.com/kyverno/kyverno-json/pkg/core/templating" reflectutils "github.com/kyverno/kyverno-json/pkg/utils/reflect" ) @@ -25,7 +24,7 @@ type Projection struct { Handler } -func Parse(in any, compiler templating.Compiler) (projection Projection) { +func Parse(in any, compiler compilers.Compilers) (projection Projection) { switch typed := in.(type) { case string: // 1. if we have a string, parse the expression diff --git a/pkg/core/projection/projection_test.go b/pkg/core/projection/projection_test.go index 202795d7..9336762d 100644 --- a/pkg/core/projection/projection_test.go +++ b/pkg/core/projection/projection_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/jmespath-community/go-jmespath/pkg/binding" - "github.com/kyverno/kyverno-json/pkg/core/templating" + "github.com/kyverno/kyverno-json/pkg/core/compilers" tassert "github.com/stretchr/testify/assert" ) @@ -88,7 +88,7 @@ func TestProjection(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler := templating.DefaultCompiler + compiler := compilers.DefaultCompiler proj := Parse(tt.key, compiler) got, found, err := proj.Handler(tt.value, tt.bindings) if tt.wantErr { diff --git a/pkg/json-engine/engine.go b/pkg/json-engine/engine.go index 2503b1dd..2af86592 100644 --- a/pkg/json-engine/engine.go +++ b/pkg/json-engine/engine.go @@ -7,7 +7,7 @@ import ( jpbinding "github.com/jmespath-community/go-jmespath/pkg/binding" "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" - "github.com/kyverno/kyverno-json/pkg/core/templating" + "github.com/kyverno/kyverno-json/pkg/core/compilers" "github.com/kyverno/kyverno-json/pkg/engine" "github.com/kyverno/kyverno-json/pkg/engine/builder" "github.com/kyverno/kyverno-json/pkg/matching" @@ -67,7 +67,7 @@ func New() engine.Engine[Request, Response] { bindings jpbinding.Bindings } compiler := matching.Compiler{ - Compiler: templating.DefaultCompiler, + Compilers: compilers.DefaultCompiler, } ruleEngine := builder. Function(func(ctx context.Context, r ruleRequest) []RuleResponse { @@ -80,7 +80,7 @@ func New() engine.Engine[Request, Response] { } identifier := "" if r.rule.Identifier != "" { - result, err := templating.Execute(r.rule.Identifier, r.resource, bindings, compiler.Compiler.Jp) + result, err := compilers.Execute(r.rule.Identifier, r.resource, bindings, compiler.Jp) if err != nil { identifier = fmt.Sprintf("(error: %s)", err) } else { @@ -119,7 +119,7 @@ func New() engine.Engine[Request, Response] { } var feedback map[string]Feedback for _, f := range r.rule.Feedback { - result, err := templating.Execute(f.Value, r.resource, bindings, compiler.Compiler.Jp) + result, err := compilers.Execute(f.Value, r.resource, bindings, compiler.Jp) if feedback == nil { feedback = map[string]Feedback{} } diff --git a/pkg/matching/compiler.go b/pkg/matching/compiler.go index 7e7349db..cd31c830 100644 --- a/pkg/matching/compiler.go +++ b/pkg/matching/compiler.go @@ -6,11 +6,11 @@ import ( "github.com/cespare/xxhash/v2" "github.com/elastic/go-freelru" "github.com/kyverno/kyverno-json/pkg/core/assertion" - "github.com/kyverno/kyverno-json/pkg/core/templating" + "github.com/kyverno/kyverno-json/pkg/core/compilers" ) type Compiler struct { - templating.Compiler + compilers.Compilers *freelru.SyncedLRU[string, func() (assertion.Assertion, error)] } @@ -19,9 +19,9 @@ func hashStringXXHASH(s string) uint32 { return uint32(sum) //nolint:gosec } -func NewCompiler(compiler templating.Compiler, cacheSize uint32) Compiler { +func NewCompiler(compiler compilers.Compilers, cacheSize uint32) Compiler { out := Compiler{ - Compiler: compiler, + Compilers: compiler, } if cache, err := freelru.NewSynced[string, func() (assertion.Assertion, error)](cacheSize, hashStringXXHASH); err == nil { out.SyncedLRU = cache @@ -31,12 +31,12 @@ func NewCompiler(compiler templating.Compiler, cacheSize uint32) Compiler { func (c Compiler) CompileAssertion(hash string, value any) (assertion.Assertion, error) { if c.SyncedLRU == nil { - return assertion.Parse(value, c.Compiler) + return assertion.Parse(value, c.Compilers) } entry, _ := c.SyncedLRU.Get(hash) if entry == nil { entry = sync.OnceValues(func() (assertion.Assertion, error) { - return assertion.Parse(value, c.Compiler) + return assertion.Parse(value, c.Compilers) }) c.SyncedLRU.Add(hash, entry) } diff --git a/pkg/server/playground/handler.go b/pkg/server/playground/handler.go index 90e342e1..c4939ae7 100644 --- a/pkg/server/playground/handler.go +++ b/pkg/server/playground/handler.go @@ -8,7 +8,7 @@ import ( "github.com/gin-gonic/gin" "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" - "github.com/kyverno/kyverno-json/pkg/core/templating" + "github.com/kyverno/kyverno-json/pkg/core/compilers" jsonengine "github.com/kyverno/kyverno-json/pkg/json-engine" "github.com/kyverno/kyverno-json/pkg/server/model" "github.com/loopfz/gadgeto/tonic" @@ -34,7 +34,7 @@ func newHandler() (gin.HandlerFunc, error) { } // apply pre processors for _, preprocessor := range in.Preprocessors { - result, err := templating.Execute(preprocessor, payload, nil, templating.DefaultCompiler.Jp) + result, err := compilers.Execute(preprocessor, payload, nil, compilers.DefaultCompiler.Jp) if err != nil { return nil, fmt.Errorf("failed to execute prepocessor (%s) - %w", preprocessor, err) } diff --git a/pkg/server/scan/handler.go b/pkg/server/scan/handler.go index 0e5915ab..194e1928 100644 --- a/pkg/server/scan/handler.go +++ b/pkg/server/scan/handler.go @@ -8,7 +8,7 @@ import ( "github.com/gin-gonic/gin" "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" - "github.com/kyverno/kyverno-json/pkg/core/templating" + "github.com/kyverno/kyverno-json/pkg/core/compilers" jsonengine "github.com/kyverno/kyverno-json/pkg/json-engine" "github.com/kyverno/kyverno-json/pkg/server/model" "github.com/loopfz/gadgeto/tonic" @@ -26,7 +26,7 @@ func newHandler(policyProvider PolicyProvider) (gin.HandlerFunc, error) { payload := in.Payload // apply pre processors for _, preprocessor := range in.Preprocessors { - result, err := templating.Execute(preprocessor, payload, nil, templating.DefaultCompiler.Jp) + result, err := compilers.Execute(preprocessor, payload, nil, compilers.DefaultCompiler.Jp) if err != nil { return nil, fmt.Errorf("failed to execute prepocessor (%s) - %w", preprocessor, err) }