From 51461a4f4b0af02461d4ec9e6c222d02a3a3dae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Fri, 20 Sep 2024 15:04:29 +0200 Subject: [PATCH] feat: add assertion compiler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché --- go.mod | 4 +- go.sum | 8 ++++ pkg/apis/policy/v1alpha1/assertion_tree.go | 27 ++++++++++--- pkg/json-engine/engine.go | 8 ++-- pkg/matching/compiler.go | 44 ++++++++++++++++++++++ pkg/matching/match.go | 17 ++++----- website/apis/config.yaml | 1 + 7 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 pkg/matching/compiler.go diff --git a/go.mod b/go.mod index 91e64778..7a41419c 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.22.2 require ( github.com/aquilax/truncate v1.0.0 github.com/blang/semver/v4 v4.0.0 + github.com/cespare/xxhash v1.1.0 + github.com/elastic/go-freelru v0.13.0 github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 github.com/google/cel-go v0.20.1 @@ -23,6 +25,7 @@ require ( gotest.tools v2.2.0+incompatible k8s.io/apimachinery v0.31.1 k8s.io/client-go v0.31.1 + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 sigs.k8s.io/kubectl-validate v0.0.5-0.20240827210056-ce13d95db263 sigs.k8s.io/yaml v1.4.0 ) @@ -135,7 +138,6 @@ require ( k8s.io/component-base v0.31.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect - k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index b775931f..e33176f4 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/IGLOU-EU/go-wildcard v1.0.3 h1:r8T46+8/9V1STciXJomTWRpPEv4nGJATDbJkdU github.com/IGLOU-EU/go-wildcard v1.0.3/go.mod h1:/qeV4QLmydCbwH0UMQJmXDryrFKJknWi/jjO8IiuQfY= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/aquilax/truncate v1.0.0 h1:UgIGS8U/aZ4JyOJ2h3xcF5cSQ06+gGBnjxH2RUHJe0U= @@ -24,6 +26,8 @@ github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -47,6 +51,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2 h1:S6Dco8FtAhEI/qkg/00H6RdEGC+MCy5GPiQ+xweNRFE= github.com/dustinkirkland/golang-petname v0.0.0-20231002161417-6a283f1aaaf2/go.mod h1:8AuBTZBRSFqEYBPYULd+NN474/zZBLP+6WeT5S9xlAc= +github.com/elastic/go-freelru v0.13.0 h1:TKKY6yCfNNNky7Pj9xZAOEpBcdNgZJfihEftOb55omg= +github.com/elastic/go-freelru v0.13.0/go.mod h1:bSdWT4M0lW79K8QbX6XY2heQYSCqD7THoYf82pT/H3I= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -237,6 +243,8 @@ github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sS github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/pkg/apis/policy/v1alpha1/assertion_tree.go b/pkg/apis/policy/v1alpha1/assertion_tree.go index 6bb15699..3bbe9daf 100644 --- a/pkg/apis/policy/v1alpha1/assertion_tree.go +++ b/pkg/apis/policy/v1alpha1/assertion_tree.go @@ -1,8 +1,10 @@ package v1alpha1 import ( + "crypto/md5" //nolint:gosec + "encoding/hex" + "github.com/kyverno/kyverno-json/pkg/core/assertion" - "github.com/kyverno/kyverno-json/pkg/core/templating" "k8s.io/apimachinery/pkg/util/json" ) @@ -12,19 +14,30 @@ import ( // AssertionTree represents an assertion tree. type AssertionTree struct { _tree any + _hash string +} + +func hash(in any) string { + if in == nil { + return "" + } + bytes, err := json.Marshal(in) + if err != nil { + return "" + } + hash := md5.Sum(bytes) //nolint:gosec + return hex.EncodeToString(hash[:]) } func NewAssertionTree(value any) AssertionTree { return AssertionTree{ _tree: value, + _hash: hash(value), } } -func (t *AssertionTree) Assertion(compiler templating.Compiler) (assertion.Assertion, error) { - if t._tree == nil { - return nil, nil - } - return assertion.Parse(t._tree, compiler) +func (t *AssertionTree) Compile(compiler func(string, any) (assertion.Assertion, error)) (assertion.Assertion, error) { + return compiler(t._hash, t._tree) } func (a *AssertionTree) MarshalJSON() ([]byte, error) { @@ -38,9 +51,11 @@ func (a *AssertionTree) UnmarshalJSON(data []byte) error { return err } a._tree = v + a._hash = hash(a._tree) return nil } func (in *AssertionTree) DeepCopyInto(out *AssertionTree) { out._tree = deepCopy(in._tree) + out._hash = in._hash } diff --git a/pkg/json-engine/engine.go b/pkg/json-engine/engine.go index 6d60bef9..7e7c029b 100644 --- a/pkg/json-engine/engine.go +++ b/pkg/json-engine/engine.go @@ -66,7 +66,9 @@ func New() engine.Engine[Request, Response] { resource any bindings jpbinding.Bindings } - compiler := templating.NewCompiler(templating.CompilerOptions{}) + compiler := matching.Compiler{ + Compiler: templating.NewCompiler(templating.CompilerOptions{}), + } ruleEngine := builder. Function(func(ctx context.Context, r ruleRequest) []RuleResponse { bindings := r.bindings.Register("$rule", jpbinding.NewBinding(r.rule)) @@ -78,7 +80,7 @@ func New() engine.Engine[Request, Response] { } identifier := "" if r.rule.Identifier != "" { - result, err := templating.ExecuteJP(r.rule.Identifier, r.resource, bindings, compiler) + result, err := templating.ExecuteJP(r.rule.Identifier, r.resource, bindings, compiler.Compiler) if err != nil { identifier = fmt.Sprintf("(error: %s)", err) } else { @@ -117,7 +119,7 @@ func New() engine.Engine[Request, Response] { } var feedback map[string]Feedback for _, f := range r.rule.Feedback { - result, err := templating.ExecuteJP(f.Value, r.resource, bindings, compiler) + result, err := templating.ExecuteJP(f.Value, r.resource, bindings, compiler.Compiler) if feedback == nil { feedback = map[string]Feedback{} } diff --git a/pkg/matching/compiler.go b/pkg/matching/compiler.go new file mode 100644 index 00000000..a830bd03 --- /dev/null +++ b/pkg/matching/compiler.go @@ -0,0 +1,44 @@ +package matching + +import ( + "sync" + + "github.com/cespare/xxhash" + "github.com/elastic/go-freelru" + "github.com/kyverno/kyverno-json/pkg/core/assertion" + "github.com/kyverno/kyverno-json/pkg/core/templating" +) + +type Compiler struct { + templating.Compiler + *freelru.SyncedLRU[string, func() (assertion.Assertion, error)] +} + +func hashStringXXHASH(s string) uint32 { + sum := xxhash.Sum64String(s) + return uint32(sum) //nolint:gosec +} + +func NewCompiler(compiler templating.Compiler, cacheSize uint32) Compiler { + out := Compiler{ + Compiler: compiler, + } + if cache, err := freelru.NewSynced[string, func() (assertion.Assertion, error)](cacheSize, hashStringXXHASH); err == nil { + out.SyncedLRU = cache + } + return out +} + +func (c Compiler) CompileAssertion(hash string, value any) (assertion.Assertion, error) { + if c.SyncedLRU == nil { + return assertion.Parse(value, c.Compiler) + } + entry, _ := c.SyncedLRU.Get(hash) + if entry == nil { + entry = sync.OnceValues(func() (assertion.Assertion, error) { + return assertion.Parse(value, c.Compiler) + }) + c.SyncedLRU.Add(hash, entry) + } + return entry() +} diff --git a/pkg/matching/match.go b/pkg/matching/match.go index 01743a17..97113451 100644 --- a/pkg/matching/match.go +++ b/pkg/matching/match.go @@ -5,7 +5,6 @@ import ( "github.com/jmespath-community/go-jmespath/pkg/binding" "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1" - "github.com/kyverno/kyverno-json/pkg/core/templating" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -37,7 +36,7 @@ func (r Results) Error() string { return strings.Join(lines, "\n") } -func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings binding.Bindings, compiler templating.Compiler) ([]Result, error) { +func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings binding.Bindings, compiler Compiler) ([]Result, error) { if len(match.Any) == 0 && len(match.All) == 0 { return nil, field.Invalid(path, match, "an empty assert is not valid") } else { @@ -46,7 +45,7 @@ func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings b path := path.Child("any") for i, assertion := range match.Any { path := path.Index(i).Child("check") - parsed, err := assertion.Check.Assertion(compiler) + parsed, err := assertion.Check.Compile(compiler.CompileAssertion) if err != nil { return fails, err } @@ -75,7 +74,7 @@ func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings b path := path.Child("all") for i, assertion := range match.All { path := path.Index(i).Child("check") - parsed, err := assertion.Check.Assertion(compiler) + parsed, err := assertion.Check.Compile(compiler.CompileAssertion) if err != nil { return fails, err } @@ -99,7 +98,7 @@ func MatchAssert(path *field.Path, match v1alpha1.Assert, actual any, bindings b } } -func Match(path *field.Path, match *v1alpha1.Match, actual any, bindings binding.Bindings, compiler templating.Compiler) (field.ErrorList, error) { +func Match(path *field.Path, match *v1alpha1.Match, actual any, bindings binding.Bindings, compiler Compiler) (field.ErrorList, error) { if match == nil || (len(match.Any) == 0 && len(match.All) == 0) { return nil, field.Invalid(path, match, "an empty match is not valid") } else { @@ -122,11 +121,11 @@ func Match(path *field.Path, match *v1alpha1.Match, actual any, bindings binding } } -func MatchAny(path *field.Path, assertions []v1alpha1.AssertionTree, actual any, bindings binding.Bindings, compiler templating.Compiler) (field.ErrorList, error) { +func MatchAny(path *field.Path, assertions []v1alpha1.AssertionTree, actual any, bindings binding.Bindings, compiler Compiler) (field.ErrorList, error) { var errs field.ErrorList for i, assertion := range assertions { path := path.Index(i) - assertion, err := assertion.Assertion(compiler) + assertion, err := assertion.Compile(compiler.CompileAssertion) if err != nil { return errs, err } @@ -142,11 +141,11 @@ func MatchAny(path *field.Path, assertions []v1alpha1.AssertionTree, actual any, return errs, nil } -func MatchAll(path *field.Path, assertions []v1alpha1.AssertionTree, actual any, bindings binding.Bindings, compiler templating.Compiler) (field.ErrorList, error) { +func MatchAll(path *field.Path, assertions []v1alpha1.AssertionTree, actual any, bindings binding.Bindings, compiler Compiler) (field.ErrorList, error) { var errs field.ErrorList for i, assertion := range assertions { path := path.Index(i) - assertion, err := assertion.Assertion(compiler) + assertion, err := assertion.Compile(compiler.CompileAssertion) if err != nil { return errs, err } diff --git a/website/apis/config.yaml b/website/apis/config.yaml index 7cee339b..d81b4f88 100644 --- a/website/apis/config.yaml +++ b/website/apis/config.yaml @@ -4,6 +4,7 @@ hiddenMemberFields: - _tree - _assertion - _message +- _hash externalPackages: - match: ^k8s\.io/apimachinery/pkg/apis/meta/v1\.Duration$