From 6955566da21490a18c568e9c9e004e719a95e9a3 Mon Sep 17 00:00:00 2001 From: Vishal Choudhary Date: Tue, 29 Aug 2023 09:04:38 +0530 Subject: [PATCH 1/3] feat: add leader election Signed-off-by: Vishal Choudhary --- configs/install.yaml | 11 +++++ controller.go | 37 +++++++++++++++++ main.go | 95 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 124 insertions(+), 19 deletions(-) create mode 100644 controller.go diff --git a/configs/install.yaml b/configs/install.yaml index 3b20305..664ee38 100644 --- a/configs/install.yaml +++ b/configs/install.yaml @@ -98,6 +98,17 @@ rules: - create - update - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/controller.go b/controller.go new file mode 100644 index 0000000..7e5c9bb --- /dev/null +++ b/controller.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" + "sync" + + "github.com/go-logr/logr" + "github.com/kyverno/kyverno/pkg/controllers" +) + +type Controller interface { + Run(context.Context, logr.Logger, *sync.WaitGroup) +} + +type controller struct { + name string + controller controllers.Controller + workers int +} + +func NewController(name string, c controllers.Controller, w int) Controller { + return controller{ + name: name, + controller: c, + workers: w, + } +} + +func (c controller) Run(ctx context.Context, logger logr.Logger, wg *sync.WaitGroup) { + wg.Add(1) + go func(logger logr.Logger) { + logger.Info("starting controller", "workers", c.workers) + defer logger.Info("controller stopped") + defer wg.Done() + c.controller.Run(ctx, c.workers) + }(logger.WithValues("name", c.name)) +} diff --git a/main.go b/main.go index 6967578..b37ec40 100644 --- a/main.go +++ b/main.go @@ -7,9 +7,13 @@ import ( "log" "net/http" "os" + "os/signal" + "sync" + "syscall" "time" "github.com/go-logr/zapr" + "github.com/kyverno/kyverno/pkg/leaderelection" "github.com/kyverno/pkg/certmanager" tlsMgr "github.com/kyverno/pkg/tls" "github.com/nirmata/kyverno-notation-verifier/kubenotation" @@ -17,8 +21,10 @@ import ( knvVerifier "github.com/nirmata/kyverno-notation-verifier/verifier" _ "github.com/notaryproject/notation-core-go/signature/cose" _ "github.com/notaryproject/notation-core-go/signature/jws" + "github.com/pkg/errors" "go.uber.org/zap" "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" @@ -96,21 +102,14 @@ func main() { log.Fatalf("failed to initialize kube client: %v", err) } + signalCtx, sdown := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer sdown() + tlsMgrConfig := &tlsMgr.Config{ - ServiceName: "kyverno-notation-aws", + ServiceName: "svc", Namespace: namespace, } - certRenewer := tlsMgr.NewCertRenewer( - zapr.NewLogger(logger), - kubeClient.CoreV1().Secrets(namespace), - CertRenewalInterval, - CAValidityDuration, - TLSValidityDuration, - "", - tlsMgrConfig, - ) - caStopCh := make(chan struct{}, 1) caInformer := NewSecretInformer(kubeClient, namespace, tlsMgr.GenerateRootCASecretName(tlsMgrConfig), resyncPeriod) go caInformer.Informer().Run(caStopCh) @@ -119,16 +118,58 @@ func main() { tlsInformer := NewSecretInformer(kubeClient, namespace, tlsMgr.GenerateTLSPairSecretName(tlsMgrConfig), resyncPeriod) go tlsInformer.Informer().Run(tlsStopCh) - certManager := certmanager.NewController( - zapr.NewLogger(logger), - caInformer, - tlsInformer, - certRenewer, - tlsMgrConfig, + le, err := leaderelection.New( + zapr.NewLogger(logger).WithName("leader-election"), + "kyverno-notation-aws", + namespace, + kubeClient, + "kyverno-notation-aws", + 2*time.Second, + func(ctx context.Context) { + + certRenewer := tlsMgr.NewCertRenewer( + zapr.NewLogger(logger), + kubeClient.CoreV1().Secrets(namespace), + CertRenewalInterval, + CAValidityDuration, + TLSValidityDuration, + "", + tlsMgrConfig, + ) + + certManager := certmanager.NewController( + zapr.NewLogger(logger), + caInformer, + tlsInformer, + certRenewer, + tlsMgrConfig, + ) + + leaderControllers := []Controller{NewController("cert-manager", certManager, 1)} + + // start leader controllers + var wg sync.WaitGroup + for _, controller := range leaderControllers { + controller.Run(signalCtx, zapr.NewLogger(logger).WithName("controllers"), &wg) + } + // wait all controllers shut down + wg.Wait() + }, + nil, ) + if err != nil { + log.Fatalf("failed to initialize leader election: %v", err) + os.Exit(1) + } + // start leader election go func() { - certManager.Run(context.TODO(), 1) + select { + case <-signalCtx.Done(): + return + default: + le.Run(signalCtx) + } }() crdSetup, err := kubenotation.Setup(zapr.NewLogger(logger), metricsAddr, probeAddr, enableLeaderElection) @@ -172,7 +213,23 @@ func main() { errsTLS := make(chan error, 1) if !flagNoTLS { tlsConf := &tls.Config{ - GetCertificate: certManager.GetCertificate, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + secret, err := tlsInformer.Lister().Secrets(tlsMgrConfig.Namespace).Get(tlsMgr.GenerateTLSPairSecretName(tlsMgrConfig)) + if err != nil { + return nil, err + } else if secret == nil { + return nil, errors.New("tls secret not found") + } else if secret.Type != corev1.SecretTypeTLS { + return nil, errors.New("secret is not a TLS secret") + } + + cert, err := tls.X509KeyPair(secret.Data[corev1.TLSCertKey], secret.Data[corev1.TLSPrivateKeyKey]) + if err != nil { + return nil, err + } + + return &cert, nil + }, } srv := &http.Server{ Addr: ":9443", From 587b4c7c4ab4235fa1cf1c50d22134e26236e81b Mon Sep 17 00:00:00 2001 From: Vishal Choudhary Date: Tue, 29 Aug 2023 14:49:42 +0530 Subject: [PATCH 2/3] feat: add using env for pod ns and name Signed-off-by: Vishal Choudhary --- configs/install.yaml | 8 ++++++++ configs/samples/kyverno-policy.yaml | 2 +- main.go | 29 +++++++++++++++++------------ utils.go | 8 ++++++++ 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/configs/install.yaml b/configs/install.yaml index 664ee38..07431fd 100644 --- a/configs/install.yaml +++ b/configs/install.yaml @@ -188,6 +188,14 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: SERVICE_NAME + value: svc + - name: DEPLOYMENT_NAME + value: kyverno-notation-aws + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name # USE IF IRSA IS NOT CONFIGURED # - name: AWS_ACCESS_KEY_ID # value: ${AWS_ACCESS_KEY_ID} diff --git a/configs/samples/kyverno-policy.yaml b/configs/samples/kyverno-policy.yaml index 0502f04..36189dc 100644 --- a/configs/samples/kyverno-policy.yaml +++ b/configs/samples/kyverno-policy.yaml @@ -22,7 +22,7 @@ spec: context: - name: tlscerts apiCall: - urlPath: "/api/v1/namespaces/kyverno-notation-aws/secrets/kyverno-notation-aws.kyverno-notation-aws.svc.tls-pair" + urlPath: "/api/v1/namespaces/kyverno-notation-aws/secrets/svc.kyverno-notation-aws.svc.tls-pair" jmesPath: "base64_decode( data.\"tls.crt\" )" - name: response apiCall: diff --git a/main.go b/main.go index b37ec40..68ae440 100644 --- a/main.go +++ b/main.go @@ -31,11 +31,16 @@ import ( ) var ( - namespace = "kyverno-notation-aws" + Namespace = getEnvWithFallback("POD_NAMESPACE", "kyverno-notation-aws") + ServiceName = getEnvWithFallback("SERVICE_NAME", "svc") + DeploymentName = getEnvWithFallback("DEPLOYMENT_NAME", "kyverno-notation-aws") + PodName = getEnvWithFallback("POD_NAME", "kyverno-notation-aws") + CertRenewalInterval = 12 * time.Hour CAValidityDuration = 365 * 24 * time.Hour TLSValidityDuration = 150 * 24 * time.Hour - resyncPeriod = 15 * time.Minute + + resyncPeriod = 15 * time.Minute ) func main() { @@ -106,30 +111,30 @@ func main() { defer sdown() tlsMgrConfig := &tlsMgr.Config{ - ServiceName: "svc", - Namespace: namespace, + ServiceName: ServiceName, + Namespace: Namespace, } caStopCh := make(chan struct{}, 1) - caInformer := NewSecretInformer(kubeClient, namespace, tlsMgr.GenerateRootCASecretName(tlsMgrConfig), resyncPeriod) + caInformer := NewSecretInformer(kubeClient, Namespace, tlsMgr.GenerateRootCASecretName(tlsMgrConfig), resyncPeriod) go caInformer.Informer().Run(caStopCh) tlsStopCh := make(chan struct{}, 1) - tlsInformer := NewSecretInformer(kubeClient, namespace, tlsMgr.GenerateTLSPairSecretName(tlsMgrConfig), resyncPeriod) + tlsInformer := NewSecretInformer(kubeClient, Namespace, tlsMgr.GenerateTLSPairSecretName(tlsMgrConfig), resyncPeriod) go tlsInformer.Informer().Run(tlsStopCh) le, err := leaderelection.New( zapr.NewLogger(logger).WithName("leader-election"), - "kyverno-notation-aws", - namespace, + DeploymentName, + Namespace, kubeClient, - "kyverno-notation-aws", + PodName, 2*time.Second, func(ctx context.Context) { certRenewer := tlsMgr.NewCertRenewer( - zapr.NewLogger(logger), - kubeClient.CoreV1().Secrets(namespace), + zapr.NewLogger(logger).WithName("tls").WithValues("pod", PodName), + kubeClient.CoreV1().Secrets(Namespace), CertRenewalInterval, CAValidityDuration, TLSValidityDuration, @@ -138,7 +143,7 @@ func main() { ) certManager := certmanager.NewController( - zapr.NewLogger(logger), + zapr.NewLogger(logger).WithName("certmanager").WithValues("pod", PodName), caInformer, tlsInformer, certRenewer, diff --git a/utils.go b/utils.go index 09cd4f5..6f2a474 100644 --- a/utils.go +++ b/utils.go @@ -1,6 +1,7 @@ package main import ( + "os" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,3 +45,10 @@ func (i *secretInformer) Informer() cache.SharedIndexInformer { func (i *secretInformer) Lister() corev1listers.SecretLister { return i.lister } + +func getEnvWithFallback(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} From 92a3fc378166d86bb83833ec810009cc31b55e72 Mon Sep 17 00:00:00 2001 From: Vishal Choudhary Date: Tue, 29 Aug 2023 18:58:45 +0530 Subject: [PATCH 3/3] feat: remove fallback from pod name and namespace env Signed-off-by: Vishal Choudhary --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 68ae440..3f9ab60 100644 --- a/main.go +++ b/main.go @@ -31,10 +31,10 @@ import ( ) var ( - Namespace = getEnvWithFallback("POD_NAMESPACE", "kyverno-notation-aws") + Namespace = os.Getenv("POD_NAMESPACE") + PodName = os.Getenv("POD_NAME") ServiceName = getEnvWithFallback("SERVICE_NAME", "svc") DeploymentName = getEnvWithFallback("DEPLOYMENT_NAME", "kyverno-notation-aws") - PodName = getEnvWithFallback("POD_NAME", "kyverno-notation-aws") CertRenewalInterval = 12 * time.Hour CAValidityDuration = 365 * 24 * time.Hour