From 77b3e6db1732f8c0233ea90f5f2a03483f46173e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Fri, 25 Oct 2024 11:26:34 +0200 Subject: [PATCH] refactor: sidecar injector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché --- .github/workflows/tests.yaml | 10 +- .vscode/launch.json | 1 - .../templates/_helpers/_deployment.tpl | 10 + .../sidecar-injector/certificates.yaml | 56 ++++-- .../sidecar-injector/deployment.yaml | 8 + .../mutating-webhook-configuration.yaml | 35 ---- .../templates/sidecar-injector/service.yaml | 16 +- charts/kyverno-envoy-plugin/values.yaml | 10 +- go.mod | 3 +- go.sum | 2 + pkg/commands/inject/command.go | 144 +++++++++++++- pkg/httpd/simpleserver.go | 70 ------- pkg/server/handlers/admission.go | 48 +++++ pkg/server/handlers/error.go | 11 ++ pkg/signals/context.go | 3 +- pkg/webhook/health.go | 10 - tests/e2e-test/chainsaw-test.yaml | 20 -- tests/e2e-test/deployment.yaml | 182 ------------------ .../sidecar-injector/chainsaw-test.yaml | 11 ++ .../e2e-test/sidecar-injector/pod-assert.yaml | 14 ++ tests/e2e-test/sidecar-injector/pod.yaml | 13 ++ 21 files changed, 318 insertions(+), 359 deletions(-) create mode 100644 charts/kyverno-envoy-plugin/templates/_helpers/_deployment.tpl delete mode 100644 charts/kyverno-envoy-plugin/templates/sidecar-injector/mutating-webhook-configuration.yaml delete mode 100644 pkg/httpd/simpleserver.go create mode 100644 pkg/server/handlers/admission.go create mode 100644 pkg/server/handlers/error.go delete mode 100644 pkg/webhook/health.go delete mode 100644 tests/e2e-test/chainsaw-test.yaml delete mode 100644 tests/e2e-test/deployment.yaml create mode 100644 tests/e2e-test/sidecar-injector/chainsaw-test.yaml create mode 100644 tests/e2e-test/sidecar-injector/pod-assert.yaml create mode 100644 tests/e2e-test/sidecar-injector/pod.yaml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ea6dde6..af1f193 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -86,16 +86,16 @@ jobs: with: go-version-file: go.mod cache-dependency-path: go.sum - - name: Run tests - run: | - set -e - make kind-create-cluster - make kind-load-taged-image - name: Install Cosign uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 - name: Install chainsaw uses: kyverno/action-install-chainsaw@d311eacde764f806c9658574ff64c9c3b21f8397 # v0.2.11 with: verify: true + - name: Setup test environment + run: | + set -e + make kind-create-cluster + make chart-install - name: Run Chainsaw Tests run: chainsaw test tests/e2e-test diff --git a/.vscode/launch.json b/.vscode/launch.json index 404706c..5506068 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,7 +19,6 @@ "program": "${workspaceFolder}", "args": [ "sidecar-injector", - "--local" ], } ] diff --git a/charts/kyverno-envoy-plugin/templates/_helpers/_deployment.tpl b/charts/kyverno-envoy-plugin/templates/_helpers/_deployment.tpl new file mode 100644 index 0000000..5898ed0 --- /dev/null +++ b/charts/kyverno-envoy-plugin/templates/_helpers/_deployment.tpl @@ -0,0 +1,10 @@ +{{/* vim: set filetype=mustache: */}} + +{{- define "kyverno.deployment.replicas" -}} + {{- if and (not (kindIs "invalid" .)) (not (kindIs "string" .)) -}} + {{- if eq (int .) 0 -}} + {{- fail "Kyverno does not support running with 0 replicas. Please provide a non-zero integer value." -}} + {{- end -}} + {{- end -}} + {{- . -}} +{{- end -}} diff --git a/charts/kyverno-envoy-plugin/templates/sidecar-injector/certificates.yaml b/charts/kyverno-envoy-plugin/templates/sidecar-injector/certificates.yaml index b95689d..a58c684 100644 --- a/charts/kyverno-envoy-plugin/templates/sidecar-injector/certificates.yaml +++ b/charts/kyverno-envoy-plugin/templates/sidecar-injector/certificates.yaml @@ -1,32 +1,56 @@ {{- if .Values.sidecarInjector.enabled -}} -{{- if .Values.sidecarInjector.certificates.selfSigned -}} {{- $ca := genCA (printf "*.%s.svc" (include "kyverno.namespace" .)) 1024 -}} {{- $svcName := (printf "%s.%s.svc" (include "kyverno.sidecar-injector.name" .) (include "kyverno.namespace" .)) -}} -{{- $cert := genSignedCert $svcName nil (list $svcName) 1024 $ca -}} +{{- $tls := genSignedCert $svcName nil (list $svcName) 1024 $ca -}} +{{- if .Values.sidecarInjector.certificates.selfSigned -}} apiVersion: v1 kind: Secret metadata: - name: {{ template "kyverno.sidecar-injector.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca + name: {{ template "kyverno.sidecar-injector.name" . }} namespace: {{ template "kyverno.namespace" . }} labels: {{- include "kyverno.sidecar-injector.labels" . | nindent 4 }} + annotations: + self-signed-cert: "true" type: kubernetes.io/tls data: - tls.key: {{ $ca.Key | b64enc }} - tls.crt: {{ $ca.Cert | b64enc }} + tls.key: {{ $tls.Key | b64enc }} + tls.crt: {{ $tls.Cert | b64enc }} + ca.crt: {{ $ca.Cert | b64enc }} +{{- end }} --- -apiVersion: v1 -kind: Secret +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration metadata: - name: {{ template "kyverno.sidecar-injector.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair - namespace: {{ template "kyverno.namespace" . }} + name: {{ template "kyverno.sidecar-injector.name" . }} labels: {{- include "kyverno.sidecar-injector.labels" . | nindent 4 }} - annotations: - self-signed-cert: "true" -type: kubernetes.io/tls -data: - tls.key: {{ $cert.Key | b64enc }} - tls.crt: {{ $cert.Cert | b64enc }} -{{- end -}} +webhooks: + - name: kyverno-envoy-sidecar.kyverno-envoy-sidecar-injector.svc + clientConfig: + service: + name: {{ template "kyverno.sidecar-injector.name" . }} + namespace: {{ template "kyverno.namespace" . }} + path: "/mutate" + caBundle: {{ $ca.Cert | b64enc }} + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + rules: + - apiGroups: + - '' + apiVersions: + - v1 + resources: + - pods + operations: + - CREATE + scope: '*' + objectSelector: + matchExpressions: + - key: kyverno-envoy-sidecar/injection + operator: In + values: + - enabled {{- end -}} diff --git a/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml b/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml index 30f9205..561cff7 100644 --- a/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml +++ b/charts/kyverno-envoy-plugin/templates/sidecar-injector/deployment.yaml @@ -74,6 +74,10 @@ spec: {{- tpl (toYaml .) $ | nindent 10 }} {{- end }} serviceAccountName: {{ template "kyverno.sidecar-injector.service-account.name" . }} + volumes: + - name: certs + secret: + secretName: {{ template "kyverno.sidecar-injector.name" . }} containers: {{- with .Values.sidecarInjector.containers.injector }} - name: injector @@ -107,5 +111,9 @@ spec: args: {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} + volumeMounts: + - name: certs + mountPath: /opt/kubernetes-sidecar-injector/certs + readOnly: true {{- end }} {{- end -}} diff --git a/charts/kyverno-envoy-plugin/templates/sidecar-injector/mutating-webhook-configuration.yaml b/charts/kyverno-envoy-plugin/templates/sidecar-injector/mutating-webhook-configuration.yaml deleted file mode 100644 index 61687e4..0000000 --- a/charts/kyverno-envoy-plugin/templates/sidecar-injector/mutating-webhook-configuration.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- if .Values.sidecarInjector.enabled -}} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: {{ template "kyverno.sidecar-injector.name" . }} - labels: - {{- include "kyverno.sidecar-injector.labels" . | nindent 4 }} -webhooks: - - name: kyverno-envoy-sidecar.kyverno-envoy-sidecar-injector.svc - clientConfig: - service: - name: {{ template "kyverno.sidecar-injector.name" . }} - namespace: {{ template "kyverno.namespace" . }} - path: "/mutate" - failurePolicy: Fail - sideEffects: None - admissionReviewVersions: - - v1 - rules: - - apiGroups: - - '' - apiVersions: - - v1 - resources: - - pods - operations: - - CREATE - scope: '*' - objectSelector: - matchExpressions: - - key: kyverno-envoy-sidecar/injection - operator: In - values: - - enabled -{{- end -}} diff --git a/charts/kyverno-envoy-plugin/templates/sidecar-injector/service.yaml b/charts/kyverno-envoy-plugin/templates/sidecar-injector/service.yaml index fa745d9..fad9969 100644 --- a/charts/kyverno-envoy-plugin/templates/sidecar-injector/service.yaml +++ b/charts/kyverno-envoy-plugin/templates/sidecar-injector/service.yaml @@ -6,13 +6,21 @@ metadata: namespace: {{ template "kyverno.namespace" . }} labels: {{- include "kyverno.sidecar-injector.labels" . | nindent 4 }} + {{- with .Values.sidecarInjector.service.annotations }} + annotations: + {{- tpl (toYaml .) $ | nindent 4 }} + {{- end }} spec: - type: ClusterIP + type: {{ .Values.sidecarInjector.service.type }} ports: - name: https + port: {{ .Values.sidecarInjector.service.port }} protocol: TCP - port: 443 - targetPort: 8443 + appProtocol: https + targetPort: https + {{- if and (eq .Values.sidecarInjector.service.type "NodePort") (not (empty .Values.sidecarInjector.service.nodePort)) }} + nodePort: {{ .Values.sidecarInjector.service.nodePort }} + {{- end }} selector: - {{- include "kyverno.sidecar-injector.labels" . | nindent 4 }} + {{- include "kyverno.sidecar-injector.labels.match" . | nindent 4 }} {{- end -}} diff --git a/charts/kyverno-envoy-plugin/values.yaml b/charts/kyverno-envoy-plugin/values.yaml index 192be52..1498e31 100644 --- a/charts/kyverno-envoy-plugin/values.yaml +++ b/charts/kyverno-envoy-plugin/values.yaml @@ -165,7 +165,7 @@ sidecarInjector: # @default -- See [values.yaml](values.yaml) startupProbe: httpGet: - path: /health/liveness + path: /livez port: 9443 scheme: HTTPS failureThreshold: 20 @@ -178,7 +178,7 @@ sidecarInjector: # @default -- See [values.yaml](values.yaml) livenessProbe: httpGet: - path: /health/liveness + path: /livez port: 9443 scheme: HTTPS initialDelaySeconds: 15 @@ -193,7 +193,7 @@ sidecarInjector: # @default -- See [values.yaml](values.yaml) readinessProbe: httpGet: - path: /health/readiness + path: /readyz port: 9443 scheme: HTTPS initialDelaySeconds: 5 @@ -211,7 +211,9 @@ sidecarInjector: # -- Container args. args: - sidecar-injector - - --port=9443 + - --address=:9443 + - --cert-file=/opt/kubernetes-sidecar-injector/certs/tls.crt + - --key-file=/opt/kubernetes-sidecar-injector/certs/tls.key service: diff --git a/go.mod b/go.mod index 1e1ff78..a2cf863 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 + gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 google.golang.org/grpc v1.67.1 k8s.io/apimachinery v0.29.3 @@ -108,7 +109,7 @@ require ( k8s.io/component-base v0.29.2 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240221221325-2ac9dc51f3f1 // indirect - k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect + k8s.io/utils v0.0.0-20240102154912-e7106e64919e sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kubectl-validate v0.0.2-0.20240102223437-fe143bcde89f // indirect diff --git a/go.sum b/go.sum index 2992202..084ee03 100644 --- a/go.sum +++ b/go.sum @@ -300,6 +300,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= diff --git a/pkg/commands/inject/command.go b/pkg/commands/inject/command.go index c727d9d..a493e62 100644 --- a/pkg/commands/inject/command.go +++ b/pkg/commands/inject/command.go @@ -1,26 +1,152 @@ package inject import ( + "context" + "crypto/tls" + "encoding/json" + "errors" "fmt" + "net/http" + "time" - "github.com/kyverno/kyverno-envoy-plugin/pkg/httpd" + "github.com/kyverno/kyverno-envoy-plugin/pkg/server/handlers" + "github.com/kyverno/kyverno-envoy-plugin/pkg/signals" "github.com/spf13/cobra" + "go.uber.org/multierr" + jsonpatch "gomodules.xyz/jsonpatch/v2" + admissionv1 "k8s.io/api/admission/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" ) func Command() *cobra.Command { - var httpdConf httpd.SimpleServer + var address string + var certFile string + var keyFile string command := &cobra.Command{ Use: "sidecar-injector", Short: "Responsible for injecting sidecars into pod containers", RunE: func(cmd *cobra.Command, args []string) error { - fmt.Printf("SimpleServer starting to listen in port %v", httpdConf.Port) - return httpdConf.Start() + return runServer(context.Background(), address, certFile, keyFile) }, } - command.Flags().IntVar(&httpdConf.Port, "port", 443, "server port.") - command.Flags().StringVar(&httpdConf.CertFile, "certFile", "/etc/mutator/certs/tls.crt", "File containing tls certificate") - command.Flags().StringVar(&httpdConf.KeyFile, "keyFile", "/etc/mutator/certs/tls.key", "File containing tls private key") - command.Flags().BoolVar(&httpdConf.Local, "local", false, "Local run mode") - command.Flags().StringVar(&(&httpdConf.Patcher).SidecarDataKey, "sidecarDataKey", "sidecars.yaml", "ConfigMap Sidecar Data Key") + command.Flags().StringVar(&address, "address", ":9443", "Address to listen on") + command.Flags().StringVar(&certFile, "cert-file", "", "File containing tls certificate") + command.Flags().StringVar(&keyFile, "key-file", "", "File containing tls private key") return command } + +func setupMux() http.Handler { + mux := http.NewServeMux() + mux.Handle("/livez", handlers.Health()) + mux.Handle("/readyz", handlers.Health()) + mux.Handle("/mutate", handlers.AdmissionReview(func(ctx context.Context, r *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse { + response := func(err error, warnings ...string) *admissionv1.AdmissionResponse { + response := admissionv1.AdmissionResponse{ + Allowed: err == nil, + UID: r.UID, + } + if err != nil { + response.Result = &metav1.Status{ + Status: metav1.StatusFailure, + Message: err.Error(), + } + } + response.Warnings = warnings + return &response + } + var object unstructured.Unstructured + if err := object.UnmarshalJSON(r.Object.Raw); err != nil { + fmt.Println("object.UnmarshalJSON") + return response(err) + } + var pod v1.Pod + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &pod); err != nil { + fmt.Println("FromUnstructured") + return response(err) + } + pod.Spec.Containers = append(pod.Spec.Containers, v1.Container{ + Name: "kyverno-envoy-plugin", + ImagePullPolicy: v1.PullIfNotPresent, + Image: "foo", + Args: []string{ + "serve", + }, + }) + if data, err := json.Marshal(&pod); err != nil { + fmt.Println("json.Marshal", string(data)) + return response(err) + } else if patch, err := jsonpatch.CreatePatch(r.Object.Raw, data); err != nil { + fmt.Println("jsonpatch.CreateMergePatch", string(data)) + return response(err) + } else if patch, err := json.Marshal(patch); err != nil { + fmt.Println("jsonpatch.CreateMergePatch", string(data)) + return response(err) + } else { + rspn := response(nil) + rspn.PatchType = ptr.To(admissionv1.PatchTypeJSONPatch) + rspn.Patch = patch + fmt.Println("ok", string(patch)) + return rspn + } + })) + return mux +} + +func setupServer(addr string) *http.Server { + return &http.Server{ + Addr: addr, + Handler: setupMux(), + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + // AEADs w/ ECDHE + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + }, + }, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + ReadHeaderTimeout: 30 * time.Second, + IdleTimeout: 5 * time.Minute, + } +} + +func runServer(ctx context.Context, addr, certFile, keyFile string) error { + var group wait.Group + server := setupServer(addr) + err := func() error { + signalsCtx, signalsCancel := signals.Context(ctx) + defer signalsCancel() + var shutdownErr error + group.StartWithContext(signalsCtx, func(ctx context.Context) { + <-ctx.Done() + fmt.Println("Shutting down server...") + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutdownCancel() + shutdownErr = server.Shutdown(shutdownCtx) + }) + fmt.Printf("Starting server at %s...\n", addr) + var serveErr error + if certFile != "" && keyFile != "" { + serveErr = server.ListenAndServeTLS(certFile, keyFile) + } else { + serveErr = server.ListenAndServe() + } + if errors.Is(serveErr, http.ErrServerClosed) { + serveErr = nil + } + return multierr.Combine(serveErr, shutdownErr) + }() + group.Wait() + fmt.Println("Server stopped") + return err +} diff --git a/pkg/httpd/simpleserver.go b/pkg/httpd/simpleserver.go deleted file mode 100644 index 2c7f761..0000000 --- a/pkg/httpd/simpleserver.go +++ /dev/null @@ -1,70 +0,0 @@ -package httpd - -import ( - "fmt" - "net/http" - "os" - "path/filepath" - - "github.com/kyverno/kyverno-envoy-plugin/pkg/admission" - "github.com/kyverno/kyverno-envoy-plugin/pkg/webhook" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" -) - -/*SimpleServer is the required config to create httpd server*/ -type SimpleServer struct { - Local bool - Port int - CertFile string - KeyFile string - Patcher webhook.SidecarInjectorPatcher - Debug bool -} - -// Start the simple http server supporting TLS -func (simpleServer *SimpleServer) Start() error { - k8sClient, err := simpleServer.CreateClient() - if err != nil { - return err - } - simpleServer.Patcher.K8sClient = k8sClient - server := &http.Server{ - Addr: fmt.Sprintf(":%d", simpleServer.Port), - } - mux := http.NewServeMux() - server.Handler = mux - admissionHandler := &admission.Handler{ - Handler: &admission.PodAdmissionRequestHandler{ - PodHandler: &simpleServer.Patcher, - }, - } - mux.HandleFunc("/healthz", webhook.HealthCheckHandler) - mux.HandleFunc("/mutate", admissionHandler.HandleAdmission) - if simpleServer.Local { - return server.ListenAndServe() - } - return server.ListenAndServeTLS(simpleServer.CertFile, simpleServer.KeyFile) -} - -// CreateClient Create the server -func (simpleServer *SimpleServer) CreateClient() (*kubernetes.Clientset, error) { - config, err := simpleServer.buildConfig() - if err != nil { - return nil, errors.Wrapf(err, "error setting up cluster config") - } - return kubernetes.NewForConfig(config) -} - -func (simpleServer *SimpleServer) buildConfig() (*rest.Config, error) { - if simpleServer.Local { - log.Debug("Using local kubeconfig.") - kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") - return clientcmd.BuildConfigFromFlags("", kubeconfig) - } - log.Debug("Using in cluster kubeconfig.") - return rest.InClusterConfig() -} diff --git a/pkg/server/handlers/admission.go b/pkg/server/handlers/admission.go new file mode 100644 index 0000000..8850589 --- /dev/null +++ b/pkg/server/handlers/admission.go @@ -0,0 +1,48 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" +) + +func AdmissionReview(inner func(context.Context, *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Body == nil { + HttpError(r.Context(), w, r, errors.New("empty body"), http.StatusBadRequest) + return + } + defer r.Body.Close() + body, err := io.ReadAll(r.Body) + if err != nil { + HttpError(r.Context(), w, r, err, http.StatusBadRequest) + return + } + contentType := r.Header.Get("Content-Type") + if contentType != "application/json" { + HttpError(r.Context(), w, r, errors.New("invalid Content-Type"), http.StatusUnsupportedMediaType) + return + } + var admissionReview admissionv1.AdmissionReview + if err := json.Unmarshal(body, &admissionReview); err != nil { + HttpError(r.Context(), w, r, err, http.StatusExpectationFailed) + return + } + admissionResponse := inner(r.Context(), admissionReview.Request) + admissionReview.Response = admissionResponse + responseJSON, err := json.Marshal(admissionReview) + if err != nil { + HttpError(r.Context(), w, r, err, http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + if _, err := w.Write(responseJSON); err != nil { + HttpError(r.Context(), w, r, err, http.StatusInternalServerError) + return + } + } +} diff --git a/pkg/server/handlers/error.go b/pkg/server/handlers/error.go new file mode 100644 index 0000000..a121134 --- /dev/null +++ b/pkg/server/handlers/error.go @@ -0,0 +1,11 @@ +package handlers + +import ( + "context" + "net/http" +) + +func HttpError(ctx context.Context, writer http.ResponseWriter, request *http.Request, err error, code int) { + // logger.Error(err, "an error has occurred", "url", request.URL.String()) + http.Error(writer, err.Error(), code) +} diff --git a/pkg/signals/context.go b/pkg/signals/context.go index f3fbccb..a5cd15c 100644 --- a/pkg/signals/context.go +++ b/pkg/signals/context.go @@ -2,11 +2,10 @@ package signals import ( "context" - "os" "os/signal" "syscall" ) func Context(ctx context.Context) (context.Context, context.CancelFunc) { - return signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + return signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) } diff --git a/pkg/webhook/health.go b/pkg/webhook/health.go deleted file mode 100644 index f015807..0000000 --- a/pkg/webhook/health.go +++ /dev/null @@ -1,10 +0,0 @@ -package webhook - -import ( - "net/http" -) - -// HealthCheckHandler HttpServer function to handle Health check -func HealthCheckHandler(writer http.ResponseWriter, _ *http.Request) { - writer.WriteHeader(http.StatusOK) -} diff --git a/tests/e2e-test/chainsaw-test.yaml b/tests/e2e-test/chainsaw-test.yaml deleted file mode 100644 index 8d35da8..0000000 --- a/tests/e2e-test/chainsaw-test.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: chainsaw.kyverno.io/v1alpha1 -kind: Test -metadata: - name: test-kyverno-envoy-plugin -spec: - steps: - - try: - - apply: - file: ./deployment.yaml - - script: - content: kubectl run test -it --rm --restart=Never --image=busybox -- wget -q --output-document - testapp.default.svc.cluster.local:8080/book - check: - # Checks if the standard error output contains the string '403'. - # This is likely used to verify that the expected HTTP 403 Forbidden response was received from the test application. - (contains($stdout, '403')): false - - script: - content: kubectl run test -it --rm --restart=Never --image=busybox -- wget -q --post-data='{"bookname":"Harry Potter", "author":"J.K. Rowling"}' --output-document - testapp.default.svc.cluster.local:8080/book - check: - (contains($stderr, 'Error')): true - (contains($stdout, '403')): true \ No newline at end of file diff --git a/tests/e2e-test/deployment.yaml b/tests/e2e-test/deployment.yaml deleted file mode 100644 index 5f1c393..0000000 --- a/tests/e2e-test/deployment.yaml +++ /dev/null @@ -1,182 +0,0 @@ -# Application Deployment with kyverno-envoy-plugin and Envoy sidecars. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: testapp -spec: - replicas: 1 - selector: - matchLabels: - app: testapp - template: - metadata: - labels: - app: testapp - spec: - initContainers: - - name: proxy-init - image: sanskardevops/proxyinit:latest - # Configure the iptables bootstrap script to redirect traffic to the - # Envoy proxy on port 8000, specify that Envoy will be running as user - # 1111, These values must match up with the configuration - # defined below for the "envoy" and "kyverno-envoy-plugin" containers. - args: ["-p", "7000", "-u", "1111"] - securityContext: - capabilities: - add: - - NET_ADMIN - runAsNonRoot: false - runAsUser: 0 - containers: - - name: test-application - image: sanskardevops/test-application:0.0.1 - ports: - - containerPort: 8080 - - name: envoy - image: envoyproxy/envoy:v1.30-latest - securityContext: - runAsUser: 1111 - imagePullPolicy: IfNotPresent - volumeMounts: - - readOnly: true - mountPath: /config - name: proxy-config - args: - - "envoy" - - "--config-path" - - "/config/envoy.yaml" - - name: kyverno-envoy-plugin - image: ko.local/github.com/kyverno/kyverno-envoy-plugin:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8000 - - containerPort: 9000 - volumeMounts: - - readOnly: true - mountPath: /policies - name: policy-files - args: - - "serve" - - "--policy=/policies/policy.yaml" - volumes: - - name: proxy-config - configMap: - name: proxy-config - - name: policy-files - configMap: - name: policy-files ---- -# Envoy Config with External Authorization filter that will query kyverno-envoy-plugin. -apiVersion: v1 -kind: ConfigMap -metadata: - name: proxy-config -data: - envoy.yaml: | - static_resources: - listeners: - - address: - socket_address: - address: 0.0.0.0 - port_value: 7000 - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: auto - stat_prefix: ingress_http - route_config: - name: local_route - virtual_hosts: - - name: backend - domains: - - "*" - routes: - - match: - prefix: "/" - route: - cluster: service - http_filters: - - name: envoy.ext_authz - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz - transport_api_version: V3 - with_request_body: - max_request_bytes: 8192 - allow_partial_message: true - failure_mode_allow: false - grpc_service: - google_grpc: - target_uri: 127.0.0.1:9000 - stat_prefix: ext_authz - timeout: 0.5s - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - clusters: - - name: service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 8080 - admin: - access_log_path: "/dev/null" - address: - socket_address: - address: 0.0.0.0 - port_value: 8001 - layered_runtime: - layers: - - name: static_layer_0 - static_layer: - envoy: - resource_limits: - listener: - example_listener_name: - connection_limit: 10000 - overload: - global_downstream_max_connections: 50000 ---- -# Example policy to enforce into kyverno-envoy-plugin sidecars. -apiVersion: v1 -kind: ConfigMap -metadata: - name: policy-files -data: - policy.yaml: | - apiVersion: json.kyverno.io/v1alpha1 - kind: ValidatingPolicy - metadata: - name: checkrequest - spec: - rules: - - name: deny-other-methods - assert: - any: - - message: "only allow GET method calls at path /book " - check: - request: - http: - method: GET - path: /book ---- -apiVersion: v1 -kind: Service -metadata: - name: testapp -spec: - type: ClusterIP - selector: - app: testapp - ports: - - port: 8080 - targetPort: 8080 \ No newline at end of file diff --git a/tests/e2e-test/sidecar-injector/chainsaw-test.yaml b/tests/e2e-test/sidecar-injector/chainsaw-test.yaml new file mode 100644 index 0000000..965c74a --- /dev/null +++ b/tests/e2e-test/sidecar-injector/chainsaw-test.yaml @@ -0,0 +1,11 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: sidecar-injector +spec: + steps: + - try: + - create: + file: ./pod.yaml + - assert: + file: ./pod-assert.yaml diff --git a/tests/e2e-test/sidecar-injector/pod-assert.yaml b/tests/e2e-test/sidecar-injector/pod-assert.yaml new file mode 100644 index 0000000..71a7fdc --- /dev/null +++ b/tests/e2e-test/sidecar-injector/pod-assert.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod + labels: + kyverno-envoy-sidecar/injection: enabled +spec: + containers: + - name: busybox + image: busybox + args: + - sleep + - 1d + - name: kyverno-envoy-plugin diff --git a/tests/e2e-test/sidecar-injector/pod.yaml b/tests/e2e-test/sidecar-injector/pod.yaml new file mode 100644 index 0000000..0c62715 --- /dev/null +++ b/tests/e2e-test/sidecar-injector/pod.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod + labels: + kyverno-envoy-sidecar/injection: enabled +spec: + containers: + - name: busybox + image: busybox + args: + - sleep + - 1d