refactor: servers

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

charts/kyverno-sidecar-injector/templates/webhook.yaml

@@ -19,7 +19,7 @@ webhooks:
       service:
         name: {{ template "sidecar-injector.name" . }}
         namespace: {{ template "kyverno.lib.namespace" . }}
-        path: "/mutate"
+        path: /mutate
       {{- if .Values.certificates.static }}
       caBundle: {{ index .Values.certificates.static.crt | b64enc }}
       {{- end }}

pkg/authz/grpc.go

@@ -0,0 +1,27 @@
+package authz
+
+import (
+	"context"
+	"net"
+
+	authv3 "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3"
+	"github.com/kyverno/kyverno-envoy-plugin/pkg/server"
+	"google.golang.org/grpc"
+)
+
+func NewGrpcServer(network, addr string) server.ServerFunc {
+	return func(ctx context.Context) error {
+		// create a server
+		s := grpc.NewServer()
+		// setup our authorization service
+		svc := &service{}
+		// register our authorization service
+		authv3.RegisterAuthorizationServer(s, svc)
+		// create a listener
+		l, err := net.Listen(network, addr)
+		if err != nil {
+			return err
+		}
+		return server.RunGrpc(ctx, s, l)
+	}
+}

pkg/authz/http.go

@@ -0,0 +1,27 @@
+package authz
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/kyverno/kyverno-envoy-plugin/pkg/server"
+	"github.com/kyverno/kyverno-envoy-plugin/pkg/server/handlers"
+)
+
+func NewHttpServer(addr string) server.ServerFunc {
+	return func(ctx context.Context) error {
+		// create mux
+		mux := http.NewServeMux()
+		// register health check
+		mux.Handle("/livez", handlers.Healthy(handlers.True))
+		// register ready check
+		mux.Handle("/readyz", handlers.Ready(handlers.True))
+		// create server
+		s := &http.Server{
+			Addr:    addr,
+			Handler: mux,
+		}
+		// run server
+		return server.RunHttp(ctx, s, "", "")
+	}
+}

pkg/authz/service.go

@@ -0,0 +1,43 @@
+package authz
+
+import (
+	"context"
+	"fmt"
+
+	authv3 "github.com/envoyproxy/go-control-plane/envoy/service/auth/v3"
+	typev3 "github.com/envoyproxy/go-control-plane/envoy/type/v3"
+	"google.golang.org/genproto/googleapis/rpc/status"
+	"google.golang.org/grpc/codes"
+)
+
+type service struct{}
+
+func (s *service) Check(ctx context.Context, req *authv3.CheckRequest) (*authv3.CheckResponse, error) {
+	if r := allow(); r != nil {
+		return r, nil
+	}
+	return deny("foo"), nil
+}
+
+func allow() *authv3.CheckResponse {
+	return &authv3.CheckResponse{
+		Status:          &status.Status{Code: int32(codes.OK)},
+		HttpResponse:    &authv3.CheckResponse_OkResponse{},
+		DynamicMetadata: nil,
+	}
+}
+
+func deny(denialReason string) *authv3.CheckResponse {
+	return &authv3.CheckResponse{
+		Status: &status.Status{
+			Code: int32(codes.PermissionDenied),
+		},
+		HttpResponse: &authv3.CheckResponse_DeniedResponse{
+			DeniedResponse: &authv3.DeniedHttpResponse{
+				Status: &typev3.HttpStatus{Code: typev3.StatusCode_Forbidden},
+				Body:   fmt.Sprintf("Request denied by Kyverno JSON engine. Reason: %s", denialReason),
+			},
+		},
+		DynamicMetadata: nil,
+	}
+}

pkg/commands/inject/command.go

@@ -2,19 +2,10 @@ package inject
 
 import (
 	"context"
-	"crypto/tls"
-	"encoding/json"
-	"net/http"
-	"time"
 
-	"github.com/kyverno/kyverno-envoy-plugin/pkg/server"
-	"github.com/kyverno/kyverno-envoy-plugin/pkg/server/handlers"
-	"github.com/kyverno/kyverno-envoy-plugin/pkg/sidecar"
 	"github.com/kyverno/kyverno-envoy-plugin/pkg/signals"
+	"github.com/kyverno/kyverno-envoy-plugin/pkg/webhook"
 	"github.com/spf13/cobra"
-	jsonpatch "gomodules.xyz/jsonpatch/v2"
-	admissionv1 "k8s.io/api/admission/v1"
-	corev1 "k8s.io/api/core/v1"
 )
 
 func Command() *cobra.Command {
@@ -26,8 +17,12 @@ func Command() *cobra.Command {
 		Use:   "sidecar-injector",
 		Short: "Responsible for injecting sidecars into pod containers",
 		RunE: func(cmd *cobra.Command, args []string) error {
+			// setup signals aware context
 			return signals.Do(context.Background(), func(ctx context.Context) error {
-				return server.Run(ctx, setupServer(address, setupMux(sidecarImage)), certFile, keyFile)
+				// create server
+				http := webhook.NewServer(address, sidecarImage, certFile, keyFile)
+				// run server
+				return http.Run(ctx)
 			})
 		},
 	}
@@ -37,47 +32,3 @@ func Command() *cobra.Command {
 	command.Flags().StringVar(&sidecarImage, "sidecar-image", "", "Image to use in sidecar")
 	return command
 }
-
-func setupMux(sidecarImage string) http.Handler {
-	mux := http.NewServeMux()
-	mux.Handle("/livez", handlers.Healthy(handlers.True))
-	mux.Handle("/readyz", handlers.Ready(handlers.True))
-	mux.Handle("/mutate", handlers.AdmissionReview(func(ctx context.Context, r *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse {
-		var pod corev1.Pod
-		if err := json.Unmarshal(r.Object.Raw, &pod); err != nil {
-			return handlers.AdmissionResponse(r, err)
-		}
-		pod = sidecar.Inject(pod, sidecar.Sidecar(sidecarImage))
-		if data, err := json.Marshal(&pod); err != nil {
-			return handlers.AdmissionResponse(r, err)
-		} else if patch, err := jsonpatch.CreatePatch(r.Object.Raw, data); err != nil {
-			return handlers.AdmissionResponse(r, err)
-		} else {
-			return handlers.AdmissionResponse(r, nil, patch...)
-		}
-	}))
-	return mux
-}
-
-func setupServer(addr string, handler http.Handler) *http.Server {
-	return &http.Server{
-		Addr:    addr,
-		Handler: handler,
-		TLSConfig: &tls.Config{
-			MinVersion: tls.VersionTLS12,
-			CipherSuites: []uint16{
-				// AEADs w/ ECDHE
-				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-				tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-				tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-				tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
-				tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
-			},
-		},
-		ReadTimeout:       30 * time.Second,
-		WriteTimeout:      30 * time.Second,
-		ReadHeaderTimeout: 30 * time.Second,
-		IdleTimeout:       5 * time.Minute,
-	}
-}

pkg/commands/serve/command.go

@@ -3,23 +3,53 @@ package serve
 import (
 	"context"
 
-	"github.com/kyverno/kyverno-envoy-plugin/pkg/server"
+	"github.com/kyverno/kyverno-envoy-plugin/pkg/authz"
+	"github.com/kyverno/kyverno-envoy-plugin/pkg/signals"
 	"github.com/spf13/cobra"
+	"go.uber.org/multierr"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 func Command() *cobra.Command {
 	var httpAddress string
 	var grpcAddress string
+	var grpcNetwork string
 	command := &cobra.Command{
 		Use:   "serve",
 		Short: "Start the kyverno-envoy-plugin server",
-		Run: func(cmd *cobra.Command, args []string) {
-			ctx := context.Background()
-			srv := server.NewServers(grpcAddress, httpAddress)
-			server.StartServers(ctx, srv)
+		RunE: func(cmd *cobra.Command, args []string) error {
+			// setup signals aware context
+			return signals.Do(context.Background(), func(ctx context.Context) error {
+				// track errors
+				var httpErr, grpcErr error
+				func(ctx context.Context) {
+					// create a wait group
+					var group wait.Group
+					// wait all tasks in the group are over
+					defer group.Wait()
+					// create a cancellable context
+					ctx, cancel := context.WithCancel(ctx)
+					// create http and grpc servers
+					http := authz.NewHttpServer(httpAddress)
+					grpc := authz.NewGrpcServer(grpcNetwork, grpcAddress)
+					// run servers
+					group.StartWithContext(ctx, func(ctx context.Context) {
+						// cancel context at the end
+						defer cancel()
+						httpErr = http.Run(ctx)
+					})
+					group.StartWithContext(ctx, func(ctx context.Context) {
+						// cancel context at the end
+						defer cancel()
+						grpcErr = grpc.Run(ctx)
+					})
+				}(ctx)
+				return multierr.Combine(httpErr, grpcErr)
+			})
 		},
 	}
-	command.Flags().StringVar(&httpAddress, "http-address", "", "Address to listen on")
-	command.Flags().StringVar(&grpcAddress, "grpc-address", "", "Address to listen on for health checks")
+	command.Flags().StringVar(&httpAddress, "http-address", ":9080", "Address to listen on for health checks")
+	command.Flags().StringVar(&grpcAddress, "grpc-address", ":9081", "Address to listen on")
+	command.Flags().StringVar(&grpcNetwork, "grpc-network", "tcp", "Network to listen on")
 	return command
 }

pkg/server/grpc.go

@@ -0,0 +1,32 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	"google.golang.org/grpc"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+func RunGrpc(ctx context.Context, server *grpc.Server, listener net.Listener) error {
+	defer fmt.Println("GRPC Server stopped")
+	// create a wait group
+	var group wait.Group
+	// wait all tasks in the group are over
+	defer group.Wait()
+	// create a cancellable context
+	ctx, cancel := context.WithCancel(ctx)
+	// cancel context at the end
+	defer cancel()
+	// shutdown server when context is cancelled
+	group.StartWithContext(ctx, func(ctx context.Context) {
+		// wait context cancelled
+		<-ctx.Done()
+		fmt.Println("GRPC Server shutting down...")
+		// gracefully shutdown server
+		server.GracefulStop()
+	})
+	fmt.Printf("GRPC Server starting at %s...\n", listener.Addr())
+	return server.Serve(listener)
+}

pkg/server/run.go → pkg/server/http.go

@@ -11,8 +11,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
-func Run(ctx context.Context, server *http.Server, certFile, keyFile string) error {
-	defer fmt.Println("Server stopped")
+func RunHttp(ctx context.Context, server *http.Server, certFile, keyFile string) error {
+	defer fmt.Println("HTTP Server stopped")
 	// track shutdown error
 	var shutdownErr error
 	// track serve error
@@ -29,14 +29,15 @@ func Run(ctx context.Context, server *http.Server, certFile, keyFile string) err
 		group.StartWithContext(ctx, func(ctx context.Context) {
 			// wait context cancelled
 			<-ctx.Done()
-			fmt.Println("Server shutting down...")
+			fmt.Println("HTTP Server shutting down...")
 			// create a context with timeout
 			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 			defer cancel()
+			// gracefully shutdown server
 			shutdownErr = server.Shutdown(ctx)
 		})
 		serve := func() error {
-			fmt.Printf("Server starting at %s...\n", server.Addr)
+			fmt.Printf("HTTP Server starting at %s...\n", server.Addr)
 			if certFile != "" && keyFile != "" {
 				// server over https
 				return server.ListenAndServeTLS(certFile, keyFile)