AV-214603 Changes for accepting Gateway with some valid and some invalid listener

ako-gateway-api/k8s/ako_init.go

@@ -274,7 +274,7 @@ func (c *GatewayController) FullSyncK8s(sync bool) error {
 			resVer := meta.GetResourceVersion()
 			objects.SharedResourceVerInstanceLister().Save(key, resVer)
 		}
-		if IsValidGateway(key, gatewayObj) {
+		if valid, _ := IsValidGateway(key, gatewayObj); valid {
 			filteredGateways = append(filteredGateways, gatewayObj)
 		}
 	}

ako-gateway-api/k8s/gateway_controller.go

@@ -17,11 +17,14 @@ package k8s
 import (
 	"fmt"
 	"reflect"
+	"sort"
 	"sync"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	discovery "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
@@ -500,13 +503,23 @@ func (c *GatewayController) SetupGatewayApiEventHandlers(numWorkers uint32) {
 				utils.AviLog.Debugf("key: %s, msg: same resource version returning", key)
 				return
 			}
-			if !IsValidGateway(key, gw) {
+			valid, allowedRoutesAll := IsValidGateway(key, gw)
+			if !valid {
 				return
 			}
+			listRoutes, err := validateReferredHTTPRoute(key, allowedRoutesAll, gw)
+			if err != nil {
+				utils.AviLog.Errorf("Validation of Referred HTTPRoutes failed due to error : %s", err.Error())
+			}
 			namespace, _, _ := cache.SplitMetaNamespaceKey(utils.ObjKey(gw))
 			bkt := utils.Bkt(namespace, numWorkers)
 			c.workqueue[bkt].AddRateLimited(key)
 			utils.AviLog.Debugf("key: %s, msg: ADD", key)
+			for _, route := range listRoutes {
+				key := lib.HTTPRoute + "/" + utils.ObjKey(route)
+				c.workqueue[bkt].AddRateLimited(key)
+				utils.AviLog.Debugf("key: %s, msg: UPDATE", key)
+			}
 		},
 		DeleteFunc: func(obj interface{}) {
 			if c.DisableSync {
@@ -542,13 +555,23 @@ func (c *GatewayController) SetupGatewayApiEventHandlers(numWorkers uint32) {
 			gw := obj.(*gatewayv1.Gateway)
 			if IsGatewayUpdated(oldGw, gw) {
 				key := lib.Gateway + "/" + utils.ObjKey(gw)
-				if !IsValidGateway(key, gw) {
+				valid, allowedRoutesAll := IsValidGateway(key, gw)
+				if !valid {
 					return
 				}
+				listRoutes, err := validateReferredHTTPRoute(key, allowedRoutesAll, gw)
+				if err != nil {
+					utils.AviLog.Errorf("Validation of Referred HTTPRoutes Failed due to error : %s", err.Error())
+				}
 				namespace, _, _ := cache.SplitMetaNamespaceKey(utils.ObjKey(gw))
 				bkt := utils.Bkt(namespace, numWorkers)
 				c.workqueue[bkt].AddRateLimited(key)
 				utils.AviLog.Debugf("key: %s, msg: UPDATE", key)
+				for _, route := range listRoutes {
+					key := lib.HTTPRoute + "/" + utils.ObjKey(route)
+					c.workqueue[bkt].AddRateLimited(key)
+					utils.AviLog.Debugf("key: %s, msg: UPDATE", key)
+				}
 			}
 		},
 	}
@@ -666,6 +689,7 @@ func (c *GatewayController) SetupGatewayApiEventHandlers(numWorkers uint32) {
 			namespace, _, _ := cache.SplitMetaNamespaceKey(utils.ObjKey(httpRoute))
 			bkt := utils.Bkt(namespace, numWorkers)
 			c.workqueue[bkt].AddRateLimited(key)
+			akogatewayapiobjects.GatewayApiLister().DeleteRouteToRouteStatusMapping(utils.ObjKey(httpRoute))
 			utils.AviLog.Debugf("key: %s, msg: DELETE", key)
 		},
 		UpdateFunc: func(old, obj interface{}) {
@@ -714,3 +738,31 @@ func validateAviConfigMap(obj interface{}) (*corev1.ConfigMap, bool) {
 	}
 	return nil, false
 }
+func validateReferredHTTPRoute(key string, allowedRoutesAll bool, gateway *gatewayv1.Gateway) ([]*gatewayv1.HTTPRoute, error) {
+	namespace := gateway.Namespace
+	if allowedRoutesAll {
+		namespace = metav1.NamespaceAll
+	}
+	hrObjs, err := akogatewayapilib.AKOControlConfig().GatewayApiInformers().HTTPRouteInformer.Lister().HTTPRoutes(namespace).List(labels.Set(nil).AsSelector())
+	httpRoutes := make([]*gatewayv1.HTTPRoute, 0)
+	if err != nil {
+		return nil, err
+	}
+	for _, httpRoute := range hrObjs {
+		for _, parentRef := range httpRoute.Spec.ParentRefs {
+			if parentRef.Name == gatewayv1.ObjectName(gateway.Name) {
+				if IsHTTPRouteConfigValid(key, httpRoute) {
+					httpRoutes = append(httpRoutes, httpRoute)
+				}
+				break
+			}
+		}
+	}
+	sort.Slice(httpRoutes, func(i, j int) bool {
+		if httpRoutes[i].GetCreationTimestamp().Unix() == httpRoutes[j].GetCreationTimestamp().Unix() {
+			return httpRoutes[i].Namespace+"/"+httpRoutes[i].Name < httpRoutes[j].Namespace+"/"+httpRoutes[j].Name
+		}
+		return httpRoutes[i].GetCreationTimestamp().Unix() < httpRoutes[j].GetCreationTimestamp().Unix()
+	})
+	return httpRoutes, nil
+}

ako-gateway-api/k8s/validator.go

@@ -50,9 +50,9 @@ func IsGatewayClassValid(key string, gatewayClass *gatewayv1.GatewayClass) bool
 	return true
 }
 
-func IsValidGateway(key string, gateway *gatewayv1.Gateway) bool {
+func IsValidGateway(key string, gateway *gatewayv1.Gateway) (bool, bool) {
 	spec := gateway.Spec
-
+	allowedRoutesAll := false
 	defaultCondition := akogatewayapistatus.NewCondition().
 		Type(string(gatewayv1.GatewayConditionAccepted)).
 		Reason(string(gatewayv1.GatewayReasonInvalid)).
@@ -76,7 +76,7 @@ func IsValidGateway(key string, gateway *gatewayv1.Gateway) bool {
 		programmedCondition.
 			SetIn(&gatewayStatus.Conditions)
 		akogatewayapistatus.Record(key, gateway, &status.Status{GatewayStatus: gatewayStatus})
-		return false
+		return false, allowedRoutesAll
 	}
 
 	// has 1 or none addresses
@@ -89,7 +89,7 @@ func IsValidGateway(key string, gateway *gatewayv1.Gateway) bool {
 			Reason(string(gatewayv1.GatewayReasonAddressNotUsable)).
 			SetIn(&gatewayStatus.Conditions)
 		akogatewayapistatus.Record(key, gateway, &status.Status{GatewayStatus: gatewayStatus})
-		return false
+		return false, allowedRoutesAll
 	}
 
 	if len(spec.Addresses) == 1 && *spec.Addresses[0].Type != "IPAddress" {
@@ -102,14 +102,21 @@ func IsValidGateway(key string, gateway *gatewayv1.Gateway) bool {
 			Reason(string(gatewayv1.GatewayReasonAddressNotUsable)).
 			SetIn(&gatewayStatus.Conditions)
 		akogatewayapistatus.Record(key, gateway, &status.Status{GatewayStatus: gatewayStatus})
-		return false
+		return false, allowedRoutesAll
 	}
 
 	gatewayStatus.Listeners = make([]gatewayv1.ListenerStatus, len(gateway.Spec.Listeners))
 
 	var validListenerCount int
 	for index := range spec.Listeners {
 		if isValidListener(key, gateway, gatewayStatus, index) {
+			if !allowedRoutesAll {
+				if spec.Listeners[index].AllowedRoutes != nil && spec.Listeners[index].AllowedRoutes.Namespaces != nil && spec.Listeners[index].AllowedRoutes.Namespaces.From != nil {
+					if string(*spec.Listeners[index].AllowedRoutes.Namespaces.From) == akogatewayapilib.AllowedRoutesNamespaceFromAll {
+						allowedRoutesAll = true
+					}
+				}
+			}
 			validListenerCount++
 		}
 	}
@@ -124,15 +131,16 @@ func IsValidGateway(key string, gateway *gatewayv1.Gateway) bool {
 		programmedCondition.
 			SetIn(&gatewayStatus.Conditions)
 		akogatewayapistatus.Record(key, gateway, &status.Status{GatewayStatus: gatewayStatus})
-		return false
+		return false, allowedRoutesAll
 	} else if validListenerCount < len(spec.Listeners) {
 		defaultCondition.
 			Reason(string(gatewayv1.GatewayReasonListenersNotValid)).
+			Status(metav1.ConditionTrue).
 			Message("Gateway contains atleast one valid listener").
 			SetIn(&gatewayStatus.Conditions)
 		akogatewayapistatus.Record(key, gateway, &status.Status{GatewayStatus: gatewayStatus})
 		utils.AviLog.Infof("key: %s, msg: Gateway %s contains atleast one valid listener", key, gateway.Name)
-		return false
+		return true, allowedRoutesAll
 	}
 
 	defaultCondition.
@@ -142,7 +150,7 @@ func IsValidGateway(key string, gateway *gatewayv1.Gateway) bool {
 		SetIn(&gatewayStatus.Conditions)
 	akogatewayapistatus.Record(key, gateway, &status.Status{GatewayStatus: gatewayStatus})
 	utils.AviLog.Infof("key: %s, msg: Gateway %s is valid", key, gateway.Name)
-	return true
+	return true, allowedRoutesAll
 }
 
 func isValidListener(key string, gateway *gatewayv1.Gateway, gatewayStatus *gatewayv1.GatewayStatus, index int) bool {
@@ -241,9 +249,8 @@ func isValidListener(key string, gateway *gatewayv1.Gateway, gatewayStatus *gate
 				return false
 			}
 			name := string(certRef.Name)
-			cs := utils.GetInformers().ClientSet
-			secretObj, err := cs.CoreV1().Secrets(gateway.ObjectMeta.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
-			if err != nil || secretObj == nil {
+			_, err := utils.GetInformers().ClientSet.CoreV1().Secrets(gateway.ObjectMeta.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
+			if err != nil {
 				utils.AviLog.Errorf("key: %s, msg: Secret specified in CertificateRef does not exist %+v/%+v", key, gateway.Name, listener.Name)
 				defaultCondition.SetIn(&gatewayStatus.Listeners[index].Conditions)
 				resolvedRefCondition.

ako-gateway-api/nodes/gateway_model.go

@@ -23,6 +23,7 @@ import (
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	akogatewayapilib "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/ako-gateway-api/lib"
+	akogatewayapiobjects "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/ako-gateway-api/objects"
 	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/lib"
 	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/nodes"
 	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/utils"
@@ -75,10 +76,20 @@ func (o *AviObjectGraph) BuildGatewayParent(gateway *gatewayv1.Gateway, key stri
 	return parentVsNode
 }
 
+func IsListenerInvalid(gwStatus *gatewayv1.GatewayStatus, listenerIndex int) bool {
+	if len(gwStatus.Listeners) > int(listenerIndex) && gwStatus.Listeners[listenerIndex].Conditions[0].Type == string(gatewayv1.ListenerConditionAccepted) && gwStatus.Listeners[listenerIndex].Conditions[0].Status == "False" {
+		return true
+	}
+	return false
+}
+
 func BuildPortProtocols(gateway *gatewayv1.Gateway, key string) []nodes.AviPortHostProtocol {
 	var portProtocols []nodes.AviPortHostProtocol
-	for _, listener := range gateway.Spec.Listeners {
-
+	gwStatus := akogatewayapiobjects.GatewayApiLister().GetGatewayToGatewayStatusMapping(gateway.Namespace + "/" + gateway.Name)
+	for i, listener := range gateway.Spec.Listeners {
+		if IsListenerInvalid(gwStatus, i) {
+			continue
+		}
 		pp := nodes.AviPortHostProtocol{Port: int32(listener.Port), Protocol: string(listener.Protocol)}
 		//TLS config on listener is present
 		if listener.TLS != nil && len(listener.TLS.CertificateRefs) > 0 {
@@ -96,7 +107,11 @@ func BuildTLSNodesForGateway(gateway *gatewayv1.Gateway, key string) []*nodes.Av
 	var tlsNodes []*nodes.AviTLSKeyCertNode
 	var ns, name string
 	cs := utils.GetInformers().ClientSet
-	for _, listener := range gateway.Spec.Listeners {
+	gwStatus := akogatewayapiobjects.GatewayApiLister().GetGatewayToGatewayStatusMapping(gateway.Namespace + "/" + gateway.Name)
+	for i, listener := range gateway.Spec.Listeners {
+		if IsListenerInvalid(gwStatus, i) {
+			continue
+		}
 		if listener.TLS != nil {
 			for _, certRef := range listener.TLS.CertificateRefs {
 				//kind is validated at ingestion
@@ -161,7 +176,11 @@ func DeleteTLSNode(key string, object *AviObjectGraph, gateway *gatewayv1.Gatewa
 	var tlsNodes []*nodes.AviTLSKeyCertNode
 	_, _, secretName := lib.ExtractTypeNameNamespace(key)
 	evhVsCertRefs := object.GetAviEvhVS()[0].SSLKeyCertRefs
-	for _, listener := range gateway.Spec.Listeners {
+	gwStatus := akogatewayapiobjects.GatewayApiLister().GetGatewayToGatewayStatusMapping(gateway.Namespace + "/" + gateway.Name)
+	for i, listener := range gateway.Spec.Listeners {
+		if IsListenerInvalid(gwStatus, i) {
+			continue
+		}
 		if listener.TLS != nil {
 			for _, certRef := range listener.TLS.CertificateRefs {
 				name := string(certRef.Name)
@@ -188,7 +207,11 @@ func AddTLSNode(key string, object *AviObjectGraph, gateway *gatewayv1.Gateway,
 	var tlsNodes []*nodes.AviTLSKeyCertNode
 	_, _, secretName := lib.ExtractTypeNameNamespace(key)
 	evhVsCertRefs := object.GetAviEvhVS()[0].SSLKeyCertRefs
-	for _, listener := range gateway.Spec.Listeners {
+	gwStatus := akogatewayapiobjects.GatewayApiLister().GetGatewayToGatewayStatusMapping(gateway.Namespace + "/" + gateway.Name)
+	for i, listener := range gateway.Spec.Listeners {
+		if IsListenerInvalid(gwStatus, i) {
+			continue
+		}
 		if listener.TLS != nil {
 			for _, certRef := range listener.TLS.CertificateRefs {
 				name := string(certRef.Name)

ako-gateway-api/nodes/gateway_model_rel.go

@@ -19,7 +19,9 @@ import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	akogatewayapilib "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/ako-gateway-api/lib"
 	akogatewayapiobjects "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/ako-gateway-api/objects"
@@ -126,8 +128,12 @@ func GatewayGetGw(namespace, name, key string) ([]string, bool) {
 	hostnames := make(map[string]string, 0)
 	var gwHostnames []string
 	//var hostnames map[string]string
+	gwStatus := akogatewayapiobjects.GatewayApiLister().GetGatewayToGatewayStatusMapping(gwNsName)
 
-	for _, listenerObj := range gwObj.Spec.Listeners {
+	for i, listenerObj := range gwObj.Spec.Listeners {
+		if IsListenerInvalid(gwStatus, i) {
+			continue
+		}
 		gwListener := akogatewayapiobjects.GatewayListenerStore{}
 		gwListener.Name = string(listenerObj.Name)
 		gwListener.Gateway = gwNsName
@@ -258,7 +264,21 @@ func HTTPRouteToGateway(namespace, name, key string) ([]string, bool) {
 	var gatewayList []string
 	var gwNsNameList []string
 	parentNameToHostnameMap := make(map[string][]string)
+	statusIndex := 0
+	httpRouteStatus := akogatewayapiobjects.GatewayApiLister().GetRouteToRouteStatusMapping(routeTypeNsName)
 	for _, parentRef := range hrObj.Spec.ParentRefs {
+		if statusIndex >= len(httpRouteStatus.Parents) {
+			break
+		}
+		if httpRouteStatus.Parents[statusIndex].ParentRef.Name != parentRef.Name {
+			continue
+		}
+		for statusIndex < len(httpRouteStatus.Parents) && (parentRef.SectionName != nil && *httpRouteStatus.Parents[statusIndex].ParentRef.SectionName != *parentRef.SectionName) {
+			statusIndex += 1
+		}
+		if httpRouteStatus.Parents[statusIndex].Conditions[0].Type == string(gatewayv1.RouteConditionAccepted) && httpRouteStatus.Parents[statusIndex].Conditions[0].Status == metav1.ConditionFalse {
+			continue
+		}
 		hostnameIntersection, _ := parentNameToHostnameMap[string(parentRef.Name)]
 		ns := namespace
 		if parentRef.Namespace != nil {
@@ -332,6 +352,7 @@ func HTTPRouteToGateway(namespace, name, key string) ([]string, bool) {
 			gwNsNameList = append(gwNsNameList, gwNsName)
 		}
 		parentNameToHostnameMap[string(parentRef.Name)] = hostnameIntersection
+		statusIndex += 1
 	}
 
 	utils.AviLog.Debugf("key: %s, msg: Gateways retrieved %s", key, gwNsNameList)

ako-gateway-api/nodes/route_validator.go

@@ -149,6 +149,17 @@ func validateParentReference(key string, httpRoute *gatewayv1.HTTPRoute, httpRou
 			*parentRefIndexInHttpRouteStatus = *parentRefIndexInHttpRouteStatus + 1
 			return err
 		}
+		if IsListenerInvalid(gwStatus, i) {
+			// listener is present in gateway but is in invalid state
+			utils.AviLog.Errorf("key: %s, msg: Matching gateway listener %s in Parent Reference is in invalid state", key, listenerName)
+			err := fmt.Errorf("Matching gateway listener is in Invalid state")
+			defaultCondition.
+				Reason(string(gatewayv1.RouteReasonPending)).
+				Message(err.Error()).
+				SetIn(&httpRouteStatus.Parents[*parentRefIndexInHttpRouteStatus].Conditions)
+			*parentRefIndexInHttpRouteStatus = *parentRefIndexInHttpRouteStatus + 1
+			return err
+		}
 		listenersForRoute = append(listenersForRoute, gateway.Spec.Listeners[i])
 	} else {
 		listenersForRoute = append(listenersForRoute, gateway.Spec.Listeners...)

ako-gateway-api/objects/gateway_store.go

@@ -50,6 +50,7 @@ func GatewayApiLister() *GWLister {
 			gatewayRouteToHTTPSPGPoolStore: objects.NewObjectMapStore(),
 			podToServiceStore:              objects.NewObjectMapStore(),
 			gatewayToStatus:                objects.NewObjectMapStore(),
+			routeToStatus:                  objects.NewObjectMapStore(),
 		}
 	})
 	return gwLister
@@ -119,6 +120,9 @@ type GWLister struct {
 
 	// namespace/gateway -> gateway Status
 	gatewayToStatus *objects.ObjectMapStore
+
+	// routeType/routeNs/routeName -> route Status
+	routeToStatus *objects.ObjectMapStore
 }
 
 type GatewayRouteKind struct {
@@ -259,6 +263,28 @@ func (g *GWLister) GetGatewayToGatewayStatusMapping(gwName string) *gatewayv1.Ga
 	return gatewayList.(*gatewayv1.GatewayStatus)
 }
 
+func (g *GWLister) UpdateRouteToRouteStatusMapping(routeTypeNamespaceName string, routeStatus *gatewayv1.RouteStatus) {
+	g.gwLock.Lock()
+	defer g.gwLock.Unlock()
+	g.routeToStatus.AddOrUpdate(routeTypeNamespaceName, routeStatus)
+}
+
+func (g *GWLister) DeleteRouteToRouteStatusMapping(routeTypeNamespaceName string) {
+	g.gwLock.Lock()
+	defer g.gwLock.Unlock()
+	g.routeToStatus.Delete(routeTypeNamespaceName)
+}
+
+func (g *GWLister) GetRouteToRouteStatusMapping(routeTypeNamespaceName string) *gatewayv1.RouteStatus {
+	g.gwLock.RLock()
+	defer g.gwLock.RUnlock()
+	found, routeList := g.routeToStatus.Get(routeTypeNamespaceName)
+	if !found {
+		return nil
+	}
+	return routeList.(*gatewayv1.RouteStatus)
+}
+
 //=====All route <-> gateway mappings go here.
 
 func (g *GWLister) GetRouteToGateway(routeTypeNsName string) (bool, []string) {