AV-203833: Initial commit for restricting fqdn

cmd/ako-main/main.go

@@ -111,6 +111,7 @@ func InitializeAKC() {
 	akoControlConfig.SetAKOBlockedNSList(lib.GetGlobalBlockedNSList())
 	akoControlConfig.SetControllerVRFContext(lib.GetControllerVRFContext())
 	akoControlConfig.SetAKOPrometheusFlag(lib.IsPrometheusEnabled())
+	akoControlConfig.SetAKOFQDNReusePolicy(strings.ToLower(os.Getenv("FQDN_REUSE_POLICY")))
 
 	var crdClient *crd.Clientset
 	var advl4Client *advl4.Clientset

helm/ako/templates/configmap.yaml

@@ -53,3 +53,4 @@ data:
   useDefaultSecretsOnly: {{ .Values.AKOSettings.useDefaultSecretsOnly | quote }}
   enablePrometheus: {{ default "false" .Values.featureGates.EnablePrometheus | quote }}
   enableEndpointSlice: {{ default "false" .Values.featureGates.EnableEndpointSlice | quote }} 
+  fqdnReusePolicy: {{ default "InterNamespaceAllowed" .Values.L7Settings.fqdnReusePolicy | quote}}

helm/ako/templates/statefulset.yaml

@@ -288,6 +288,11 @@ spec:
               configMapKeyRef:
                 name: avi-k8s-config
                 key: enableEndpointSlice
+          - name: FQDN_REUSE_POLICY
+            valueFrom:
+              configMapKeyRef:
+                name: avi-k8s-config
+                key: fqdnReusePolicy
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           livenessProbe:

helm/ako/values.yaml

@@ -92,6 +92,7 @@ L7Settings:
   shardVSSize: "LARGE" # Use this to control the layer 7 VS numbers. This applies to both secure/insecure VSes but does not apply for passthrough. ENUMs: LARGE, MEDIUM, SMALL, DEDICATED
   passthroughShardSize: "SMALL" # Control the passthrough virtualservice numbers using this ENUM. ENUMs: LARGE, MEDIUM, SMALL
   enableMCI: "false" # Enabling this flag would tell AKO to start processing multi-cluster ingress objects.
+  fqdnReusePolicy: "InterNamespaceAllowed" # Use this to control whether AKO allows cross-namespace usage of FQDNs. enum Strict|InterNamespaceAllowed
 
 ### This section outlines all the knobs  used to control Layer 4 loadbalancing settings in AKO.
 L4Settings:

internal/cache/controller_obj_cache.go

@@ -27,6 +27,7 @@ import (
 	"sync"
 
 	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/lib"
+	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/objects"
 	akov1beta1 "github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/apis/ako/v1beta1"
 
 	pq "github.com/jupp0r/go-priority-queue"
@@ -2186,8 +2187,17 @@ func (c *AviObjCache) AviObjVSCachePopulate(client *clients.AviClient, cloud str
 				if err := json.Unmarshal([]byte(svc_mdata_intf.(string)),
 					&svc_mdata_obj); err != nil {
 					utils.AviLog.Warnf("Error parsing service metadata during vs cache :%v", err)
+				} else if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict {
+					// call this only when FQDN policy is strict
+					hostToIngMapping := svc_mdata_obj.HostToNamespaceIngressName
+					utils.AviLog.Debugf("HosttoIng mapping is %v", utils.Stringify(hostToIngMapping))
+					if hostToIngMapping != nil {
+						// Now populate the map
+						PopulateHostToIngMapping(hostToIngMapping)
+					}
 				}
 			}
+
 			var sni_child_collection []string
 			vh_child, found := vs["vh_child_vs_uuid"]
 			if found {
@@ -2397,7 +2407,6 @@ func (c *AviObjCache) AviObjVSCachePopulate(client *clients.AviClient, cloud str
 				}
 				c.VsCacheLocal.AviCacheAdd(k, &vsMetaObj)
 				utils.AviLog.Debugf("Added VS cache key :%s", utils.Stringify(&vsMetaObj))
-
 			}
 		}
 		if resp["next"] != nil {
@@ -2415,6 +2424,48 @@ func (c *AviObjCache) AviObjVSCachePopulate(client *clients.AviClient, cloud str
 	return nil
 }
 
+// Upfront populate mapping so that during FullSyncK8s, it will be used to assign ingresses/routes to appropriate hosts list.
+func PopulateHostToIngMapping(hostsToIng map[string][]string) {
+	isRoute := false
+	if utils.GetInformers().RouteInformer != nil {
+		isRoute = true
+	}
+	var routeNamespaceName objects.RouteNamspaceName
+	for host, ings := range hostsToIng {
+		// append each ingress in active list
+		utils.AviLog.Debugf("Populating Ingress mapping for host : %s", host)
+		for _, ing := range ings {
+			namespace, _, name := lib.ExtractTypeNameNamespace(ing)
+			// Fetch ingress using clientset. From informer couldn't fetch it.
+			if !isRoute {
+				ingObj, err := utils.GetInformers().IngressInformer.Lister().Ingresses(namespace).Get(name)
+				if err != nil {
+					utils.AviLog.Errorf("Unable to retrieve the ingress %s/%s during populating host to ingress map in populate cache: %s", namespace, name, err)
+					continue
+				}
+				routeNamespaceName = objects.RouteNamspaceName{
+					RouteNSRouteName: utils.Ingress + "/" + ing,
+					CreationTime:     ingObj.CreationTimestamp,
+				}
+			} else {
+				routeObj, err := utils.GetInformers().RouteInformer.Lister().Routes(namespace).Get(name)
+				if err != nil {
+					utils.AviLog.Errorf("Unable to retrieve the ingress %s/%s during populating host to ingress map in populate cache: %s", namespace, name, err)
+					continue
+				}
+				routeNamespaceName = objects.RouteNamspaceName{
+					RouteNSRouteName: utils.OshiftRoute + "/" + ing,
+					CreationTime:     routeObj.CreationTimestamp,
+				}
+			}
+
+			// Add it to the structure
+			objects.SharedUniqueNamespaceLister().UpdateHostnameToRoute(host, routeNamespaceName)
+
+		}
+	}
+}
+
 func (c *AviObjCache) AviObjOneVSCachePopulate(client *clients.AviClient, cloud string, vsName, tenant string) error {
 	// This method should be called only from layer-3 during a retry.
 	var rest_response interface{}
@@ -2458,6 +2509,14 @@ func (c *AviObjCache) AviObjOneVSCachePopulate(client *clients.AviClient, cloud
 				if err := json.Unmarshal([]byte(svc_mdata_intf.(string)),
 					&svc_mdata_obj); err != nil {
 					utils.AviLog.Warnf("Error parsing service metadata during vs cache :%v", err)
+				} else if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict {
+					// call this only when FQDN policy is strict
+					hostToIngMapping := svc_mdata_obj.HostToNamespaceIngressName
+					utils.AviLog.Debugf("HosttoIng mapping is %v", utils.Stringify(hostToIngMapping))
+					if hostToIngMapping != nil {
+						// Now populate the map
+						PopulateHostToIngMapping(hostToIngMapping)
+					}
 				}
 			}
 			var sni_child_collection []string

internal/k8s/ako_init.go

@@ -530,16 +530,15 @@ func (c *AviController) InitController(informers K8sinformers, registeredInforme
 	statusQueueParams := utils.WorkerQueue{NumWorkers: numGraphWorkers, WorkqueueName: utils.StatusQueue}
 	graphQueue = utils.SharedWorkQueue(&ingestionQueueParams, &graphQueueParams, &slowRetryQParams, &fastRetryQParams, &statusQueueParams).GetQueueByName(utils.GraphLayer)
 
+	c.addIndexers()
+	c.Start(stopCh)
 	err := PopulateCache()
 	if err != nil {
 		c.DisableSync = true
 		utils.AviLog.Errorf("failed to populate cache, disabling sync")
 		lib.ShutdownApi()
 	}
 
-	c.addIndexers()
-	c.Start(stopCh)
-
 	fullSyncInterval := os.Getenv(utils.FULL_SYNC_INTERVAL)
 	interval, err := strconv.ParseInt(fullSyncInterval, 10, 64)
 
@@ -1100,36 +1099,69 @@ func (c *AviController) FullSyncK8s(sync bool) error {
 		}
 		//Ingress Section
 		if utils.GetInformers().IngressInformer != nil {
+			ingObjList := make([]*networkingv1.Ingress, 0)
+			// create list of ingresses.
 			for namespace := range acceptedNamespaces {
 				ingObjs, err := utils.GetInformers().IngressInformer.Lister().Ingresses(namespace).List(labels.Set(nil).AsSelector())
 				if err != nil {
 					utils.AviLog.Errorf("Unable to retrieve the ingresses during full sync: %s", err)
-				} else {
-					for _, ingObj := range ingObjs {
-						key := utils.Ingress + "/" + utils.ObjKey(ingObj)
-						meta, err := meta.Accessor(ingObj)
-						if err == nil {
-							resVer := meta.GetResourceVersion()
-							objects.SharedResourceVerInstanceLister().Save(key, resVer)
-						}
-						utils.AviLog.Debugf("Dequeue for ingress key: %v", key)
-						lib.IncrementQueueCounter(utils.ObjectIngestionLayer)
-						nodes.DequeueIngestion(key, true)
+					continue
+				}
+				ingObjList = append(ingObjList, ingObjs...)
+			}
+			// sort the list as per the timestamp. Sorting logic can be kept irrespective of strict or non-strict policy
+			sort.Slice(ingObjList, func(i, j int) bool { return lib.IngressLessthan(ingObjList[i], ingObjList[j]) })
+
+			for _, ingObj := range ingObjList {
+				key := utils.Ingress + "/" + ingObj.Namespace + "/" + ingObj.Name
+				isValid := true
+
+				if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict {
+					// get the hostnames in the ingress
+					isValid, _ = isIngAcceptedWithFQDNRestriction(key, ingObj)
+				}
+				if isValid {
+					meta, err := meta.Accessor(ingObj)
+					if err == nil {
+						resVer := meta.GetResourceVersion()
+						objects.SharedResourceVerInstanceLister().Save(key, resVer)
 					}
+					utils.AviLog.Debugf("Dequeue for ingress key: %v", key)
+					lib.IncrementQueueCounter(utils.ObjectIngestionLayer)
+					nodes.DequeueIngestion(key, true)
+				} else {
+					utils.AviLog.Warnf("key: %s, msg: Ingress is not accepted due to FQDN restriction policy", key)
 				}
 			}
+			// TODO: free ingObjList
 		}
 		//Route Section
 		if utils.GetInformers().RouteInformer != nil {
-			routeObjs, err := utils.GetInformers().RouteInformer.Lister().List(labels.Set(nil).AsSelector())
-			if err != nil {
-				utils.AviLog.Errorf("Unable to retrieve the routes during full sync: %s", err)
-			} else {
-				for _, routeObj := range routeObjs {
-					if _, ok := acceptedNamespaces[routeObj.Namespace]; !ok {
-						continue
+			routeObjList := make([]*routev1.Route, 0)
+			for namespace := range acceptedNamespaces {
+				routeObjs, err := utils.GetInformers().RouteInformer.Lister().Routes(namespace).List(labels.Set(nil).AsSelector())
+				if err != nil {
+					utils.AviLog.Errorf("Unable to retrieve the ingresses during full sync: %s", err)
+					continue
+				}
+				routeObjList = append(routeObjList, routeObjs...)
+			}
+
+			// sort on timestamp
+			sort.Slice(routeObjList, func(i, j int) bool { return lib.RouteLessthan(routeObjList[i], routeObjList[j]) })
+
+			for _, routeObj := range routeObjList {
+				key := utils.OshiftRoute + "/" + utils.ObjKey(routeObj)
+				isValid := true
+
+				if lib.AKOControlConfig().GetAKOFQDNReusePolicy() == lib.FQDNReusePolicyStrict {
+					isValid = isRouteAcceptedWithFQDNRestriction(key, routeObj)
+					if isValid {
+						utils.AviLog.Debugf("Route %s is added to active list. Enqueuing it", key)
 					}
-					key := utils.OshiftRoute + "/" + utils.ObjKey(routeObj)
+				}
+				if isValid {
+					// Enqueue it only in case of valid
 					meta, err := meta.Accessor(routeObj)
 					if err == nil {
 						resVer := meta.GetResourceVersion()
@@ -1138,8 +1170,11 @@ func (c *AviController) FullSyncK8s(sync bool) error {
 					utils.AviLog.Debugf("Dequeue for route key: %v", key)
 					lib.IncrementQueueCounter(utils.ObjectIngestionLayer)
 					nodes.DequeueIngestion(key, true)
+				} else {
+					utils.AviLog.Warnf("key: %s, msg: Route is not accepted due to FQDN restriction policy", key)
 				}
 			}
+			// TODO: Free routelist
 		}
 		if lib.UseServicesAPI() {
 			gatewayObjs, err := lib.AKOControlConfig().SvcAPIInformers().GatewayInformer.Lister().Gateways(metav1.NamespaceAll).List(labels.Set(nil).AsSelector())