Skip to content

Commit

Permalink
refactor: sidecar injector
Browse files Browse the repository at this point in the history
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
  • Loading branch information
eddycharly committed Oct 25, 2024
1 parent 5a72331 commit 50998d9
Show file tree
Hide file tree
Showing 18 changed files with 258 additions and 358 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -86,16 +86,16 @@ jobs:
with:
go-version-file: go.mod
cache-dependency-path: go.sum
- name: Run tests
run: |
set -e
make kind-create-cluster
make kind-load-taged-image
- name: Install Cosign
uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0
- name: Install chainsaw
uses: kyverno/action-install-chainsaw@d311eacde764f806c9658574ff64c9c3b21f8397 # v0.2.11
with:
verify: true
- name: Setup test environment
run: |
set -e
make kind-create-cluster
make chart-install
- name: Run Chainsaw Tests
run: chainsaw test tests/e2e-test
1 change: 0 additions & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
"program": "${workspaceFolder}",
"args": [
"sidecar-injector",
"--local"
],
}
]
Expand Down
10 changes: 10 additions & 0 deletions charts/kyverno-envoy-plugin/templates/_helpers/_deployment.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{{/* vim: set filetype=mustache: */}}

{{- define "kyverno.deployment.replicas" -}}
{{- if and (not (kindIs "invalid" .)) (not (kindIs "string" .)) -}}
{{- if eq (int .) 0 -}}
{{- fail "Kyverno does not support running with 0 replicas. Please provide a non-zero integer value." -}}
{{- end -}}
{{- end -}}
{{- . -}}
{{- end -}}
Original file line number Diff line number Diff line change
@@ -1,32 +1,56 @@
{{- if .Values.sidecarInjector.enabled -}}
{{- if .Values.sidecarInjector.certificates.selfSigned -}}
{{- $ca := genCA (printf "*.%s.svc" (include "kyverno.namespace" .)) 1024 -}}
{{- $svcName := (printf "%s.%s.svc" (include "kyverno.sidecar-injector.name" .) (include "kyverno.namespace" .)) -}}
{{- $cert := genSignedCert $svcName nil (list $svcName) 1024 $ca -}}
{{- $tls := genSignedCert $svcName nil (list $svcName) 1024 $ca -}}
{{- if .Values.sidecarInjector.certificates.selfSigned -}}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "kyverno.sidecar-injector.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca
name: {{ template "kyverno.sidecar-injector.name" . }}
namespace: {{ template "kyverno.namespace" . }}
labels:
{{- include "kyverno.sidecar-injector.labels" . | nindent 4 }}
annotations:
self-signed-cert: "true"
type: kubernetes.io/tls
data:
tls.key: {{ $ca.Key | b64enc }}
tls.crt: {{ $ca.Cert | b64enc }}
tls.key: {{ $tls.Key | b64enc }}
tls.crt: {{ $tls.Cert | b64enc }}
ca.crt: {{ $ca.Cert | b64enc }}
{{- end }}
---
apiVersion: v1
kind: Secret
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
name: {{ template "kyverno.sidecar-injector.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair
namespace: {{ template "kyverno.namespace" . }}
name: {{ template "kyverno.sidecar-injector.name" . }}
labels:
{{- include "kyverno.sidecar-injector.labels" . | nindent 4 }}
annotations:
self-signed-cert: "true"
type: kubernetes.io/tls
data:
tls.key: {{ $cert.Key | b64enc }}
tls.crt: {{ $cert.Cert | b64enc }}
{{- end -}}
webhooks:
- name: kyverno-envoy-sidecar.kyverno-envoy-sidecar-injector.svc
clientConfig:
service:
name: {{ template "kyverno.sidecar-injector.name" . }}
namespace: {{ template "kyverno.namespace" . }}
path: "/mutate"
caBundle: {{ $ca.Cert | b64enc }}
failurePolicy: Fail
sideEffects: None
admissionReviewVersions:
- v1
rules:
- apiGroups:
- ''
apiVersions:
- v1
resources:
- pods
operations:
- CREATE
scope: '*'
objectSelector:
matchExpressions:
- key: kyverno-envoy-sidecar/injection
operator: In
values:
- enabled
{{- end -}}
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ spec:
{{- tpl (toYaml .) $ | nindent 10 }}
{{- end }}
serviceAccountName: {{ template "kyverno.sidecar-injector.service-account.name" . }}
volumes:
- name: certs
secret:
secretName: {{ template "kyverno.sidecar-injector.name" . }}
containers:
{{- with .Values.sidecarInjector.containers.injector }}
- name: injector
Expand Down Expand Up @@ -107,5 +111,9 @@ spec:
args:
{{- tpl (toYaml .) $ | nindent 12 }}
{{- end }}
volumeMounts:
- name: certs
mountPath: /opt/kubernetes-sidecar-injector/certs
readOnly: true
{{- end }}
{{- end -}}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,21 @@ metadata:
namespace: {{ template "kyverno.namespace" . }}
labels:
{{- include "kyverno.sidecar-injector.labels" . | nindent 4 }}
{{- with .Values.sidecarInjector.service.annotations }}
annotations:
{{- tpl (toYaml .) $ | nindent 4 }}
{{- end }}
spec:
type: ClusterIP
type: {{ .Values.sidecarInjector.service.type }}
ports:
- name: https
port: {{ .Values.sidecarInjector.service.port }}
protocol: TCP
port: 443
targetPort: 8443
appProtocol: https
targetPort: https
{{- if and (eq .Values.sidecarInjector.service.type "NodePort") (not (empty .Values.sidecarInjector.service.nodePort)) }}
nodePort: {{ .Values.sidecarInjector.service.nodePort }}
{{- end }}
selector:
{{- include "kyverno.sidecar-injector.labels" . | nindent 4 }}
{{- include "kyverno.sidecar-injector.labels.match" . | nindent 4 }}
{{- end -}}
10 changes: 6 additions & 4 deletions charts/kyverno-envoy-plugin/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ sidecarInjector:
# @default -- See [values.yaml](values.yaml)
startupProbe:
httpGet:
path: /health/liveness
path: /livez
port: 9443
scheme: HTTPS
failureThreshold: 20
Expand All @@ -178,7 +178,7 @@ sidecarInjector:
# @default -- See [values.yaml](values.yaml)
livenessProbe:
httpGet:
path: /health/liveness
path: /livez
port: 9443
scheme: HTTPS
initialDelaySeconds: 15
Expand All @@ -193,7 +193,7 @@ sidecarInjector:
# @default -- See [values.yaml](values.yaml)
readinessProbe:
httpGet:
path: /health/readiness
path: /readyz
port: 9443
scheme: HTTPS
initialDelaySeconds: 5
Expand All @@ -211,7 +211,9 @@ sidecarInjector:
# -- Container args.
args:
- sidecar-injector
- --port=9443
- --address=:9443
- --cert-file=/opt/kubernetes-sidecar-injector/certs/tls.crt
- --key-file=/opt/kubernetes-sidecar-injector/certs/tls.key

service:

Expand Down
104 changes: 95 additions & 9 deletions pkg/commands/inject/command.go
Original file line number Diff line number Diff line change
@@ -1,26 +1,112 @@
package inject

import (
"context"
"crypto/tls"
"errors"
"fmt"
"net/http"
"time"

"github.com/kyverno/kyverno-envoy-plugin/pkg/httpd"
"github.com/kyverno/kyverno-envoy-plugin/pkg/server/handlers"
"github.com/kyverno/kyverno-envoy-plugin/pkg/signals"
"github.com/spf13/cobra"
"go.uber.org/multierr"
admissionv1 "k8s.io/api/admission/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
)

func Command() *cobra.Command {
var httpdConf httpd.SimpleServer
var address string
var certFile string
var keyFile string
command := &cobra.Command{
Use: "sidecar-injector",
Short: "Responsible for injecting sidecars into pod containers",
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Printf("SimpleServer starting to listen in port %v", httpdConf.Port)
return httpdConf.Start()
return runServer(context.Background(), address, certFile, keyFile)
},
}
command.Flags().IntVar(&httpdConf.Port, "port", 443, "server port.")
command.Flags().StringVar(&httpdConf.CertFile, "certFile", "/etc/mutator/certs/tls.crt", "File containing tls certificate")
command.Flags().StringVar(&httpdConf.KeyFile, "keyFile", "/etc/mutator/certs/tls.key", "File containing tls private key")
command.Flags().BoolVar(&httpdConf.Local, "local", false, "Local run mode")
command.Flags().StringVar(&(&httpdConf.Patcher).SidecarDataKey, "sidecarDataKey", "sidecars.yaml", "ConfigMap Sidecar Data Key")
command.Flags().StringVar(&address, "address", ":9443", "Address to listen on")
command.Flags().StringVar(&certFile, "cert-file", "", "File containing tls certificate")
command.Flags().StringVar(&keyFile, "key-file", "", "File containing tls private key")
return command
}

func setupMux() http.Handler {
mux := http.NewServeMux()
mux.Handle("/livez", handlers.Health())
mux.Handle("/readyz", handlers.Health())
mux.Handle("/mutate", handlers.AdmissionReview(func(ctx context.Context, r *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse {
var err error
var warnings []string
response := admissionv1.AdmissionResponse{
Allowed: err == nil,
UID: r.UID,
}
if err != nil {
response.Result = &metav1.Status{
Status: metav1.StatusFailure,
Message: err.Error(),
}
}
response.Warnings = warnings
return &response
}))
return mux
}

func setupServer(addr string) *http.Server {
return &http.Server{
Addr: addr,
Handler: setupMux(),
TLSConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
CipherSuites: []uint16{
// AEADs w/ ECDHE
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
},
},
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
ReadHeaderTimeout: 30 * time.Second,
IdleTimeout: 5 * time.Minute,
}
}

func runServer(ctx context.Context, addr, certFile, keyFile string) error {
var group wait.Group
server := setupServer(addr)
err := func() error {
signalsCtx, signalsCancel := signals.Context(ctx)
defer signalsCancel()
var shutdownErr error
group.StartWithContext(signalsCtx, func(ctx context.Context) {
<-ctx.Done()
fmt.Println("Shutting down server...")
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer shutdownCancel()
shutdownErr = server.Shutdown(shutdownCtx)
})
fmt.Printf("Starting server at %s...\n", addr)
var serveErr error
if certFile != "" && keyFile != "" {
serveErr = server.ListenAndServeTLS(certFile, keyFile)
} else {
serveErr = server.ListenAndServe()
}
if errors.Is(serveErr, http.ErrServerClosed) {
serveErr = nil
}
return multierr.Combine(serveErr, shutdownErr)
}()
group.Wait()
fmt.Println("Server stopped")
return err
}
Loading

0 comments on commit 50998d9

Please sign in to comment.