Merge pull request #5 from christian-vdz/feat/hsdo-client/multiple_ba…

…ckend_configuration

 feat(hsdo): add support for multiple backend configuration

README.md

@@ -1,5 +1,5 @@
 # haproxy-service-discovery-orchestrator
-Orchestrate Service Discovery for HAProxy, using the Runtime API.
+Orchestrated Service Discovery for HAProxy, using the Runtime API.
 
 No server creation/deletion, only modifications on-the-fly: no file management or reloading.
 
@@ -82,9 +82,7 @@ Configuration that is specific to each HSDO Client, next to HAProxy.
 
 `CLIENT_HAPROXY_SOCKET_PATH`: HAProxy socket to use [Runtime API](https://cbonte.github.io/haproxy-dconv/2.2/management.html#9.3). Default to `/var/run/haproxy/admin.sock`.
 
-`CLIENT_HAPROXY_BACKEND_NAME`: HAProxy default backend name. Default to ` `.
-
-`CLIENT_HAPROXY_BACKEND_BASE_NAME`: HAProxy default backend base name for server template. Default to ` `.
+`CLIENT_HAPROXY_BACKEND_LIST`: HAProxy backend list. Default to ` `.
 
 `CLIENT_HAPROXY_BACKEND_SERVER_PORT`: Port of target servers. Default to `80`.
 
@@ -111,11 +109,11 @@ You will have this kind of statistic page :
 
 `DEBUG`: To enable debug log. Default to `false`.
 
-`DYNAMODB_TABLE_NAME`: Name of dynamodb table. Default to ` `.
+`DYNAMODB_TABLE_NAME`: Name of Dynamodb table. Default to ` `.
 
-`AWS_DEFAULT_REGION`: default region needed for dynamodb access. Default to ` `.
+`AWS_DEFAULT_REGION`: default region needed for Dynamodb access. Default to ` `.
 
-`EXPORTER_PORT`: port for prometheus exporter. Default to `6789`
+`EXPORTER_PORT`: port for Prometheus exporter. Default to `6789`
 
 ## Dedicated ASG Configuration (AWS Only)
 

src/client/client.py

@@ -32,12 +32,12 @@ def run(self):
             oldDynamodbServers = dynamodbServers
             dynamodbServers = self.dynamodb.listServers()
 
-            # If dynamodb server is removed then remove metric
+            # Remove metric if dynamodb server is removed
             for oldServer in oldDynamodbServers:
                 if not self.isServerInList(oldServer, dynamodbServers):
                     self.logger.info("Removed : " + oldServer.toString())
                     Prometheus().removeMetric(oldServer)
-            # If dynamodb server is added then display metric
+            # Display metric if dynamodb server is added
             for dServer in dynamodbServers:
                 if not self.isServerInList(dServer, oldDynamodbServers) and dServer.backendServerStatus != "disabled":
                     self.logger.info("Added : " + dServer.toString())

src/client/haproxy.py

@@ -7,7 +7,7 @@
 from common.server_model import ServerModel
 
 ##
-# Configure HAProxy throught Runtime API
+# Configure HAProxy through Runtime API
 ##
 class HAProxy:
     ##
@@ -17,10 +17,20 @@ def __init__(self):
         self.azLimiter = Configuration().get("CLIENT_DEDICATED_ASG")
         self.optAllServersInFallback = Configuration().get("CLIENT_ALL_SERVERS_IN_FALLBACK_BACKEND")
         self.socketPath = Configuration().get("CLIENT_HAPROXY_SOCKET_PATH")
-        self.backendName = Configuration().get("CLIENT_HAPROXY_BACKEND_NAME")
+        self.backendList = Configuration().get("CLIENT_HAPROXY_BACKEND_LIST")
+        ## Example backendList structure:
+        # self.backendList = {
+        #     "backend-web": {
+        #         "baseName": "web-backend",
+        #         "serverPort": "80"
+        #     },
+        #     "backend-api": {
+        #         "baseName": "api-backend",
+        #         "serverPort": "8080"
+        #     }
+        # }
         self.backendServerPort = str(Configuration().get("CLIENT_HAPROXY_BACKEND_SERVER_PORT"))
         self.fallbackBackendName = Configuration().get("CLIENT_HAPROXY_FALLBACK_BACKEND_NAME")
-        self.backendBaseName = Configuration().get("CLIENT_HAPROXY_BACKEND_BASE_NAME")
         self.fallbackBackendBaseName = Configuration().get("CLIENT_HAPROXY_FALLBACK_BACKEND_BASE_NAME")
         self.ASG = Configuration().get("CLIENT_ASG_NAMES").split(",")
         self.logger = Logger("HSDO.client.haproxy")
@@ -45,22 +55,26 @@ def checkBackendConf(self):
 
     def backendConfReady(self, stat):
         stat = stat.split("\n")
-        backendExists = False
+        backendsExist = False
+        existingBackends = 0
         fallbackBackendExists = False
         for backend in stat:
             values = backend.split(",")
             ## Example backend line
             ## http_back,mywebapp2,0,0,0,0,,0,0,0,,0,,0,0,0,0,UP,10,1,0,0,1,330587,0,,1,4,2,,0,,2,0,,0,L4OK,,0,0,0,0,0,0,0,,,,,0,0,,,,,-1,,,0,0,0,0,,,,Layer4 check passed,,2,3,4,,,,10.14.34.198:80,,http,,,,,,,,0,0,0,,,0,,0,0,0,0,
-            if len(values) > 80 and values[0] == self.backendName and values[1].startswith(self.backendBaseName):
-                backendExists = True
-            elif self.azLimiter == "true" and len(values) > 80 and values[0] == self.fallbackBackendName and values[1].startswith(self.fallbackBackendBaseName):
+            for k,v in self.backendList.items():
+                if values[0] == k and values[1].startswith(v["baseName"]):
+                    existingBackends += 1
+            if self.azLimiter == "true" and len(values) > 80 and values[0] == self.fallbackBackendName and values[1].startswith(self.fallbackBackendBaseName):
                 fallbackBackendExists = True
+            if existingBackends == len(self.backendList):
+                backendsExist = True
 
         result = True
         message = ""
-        if not backendExists:
+        if not backendsExist:
             result = False
-            message = "Backend %s/%s not found, please set env CLIENT_HAPROXY_BACKEND_NAME and CLIENT_HAPROXY_BACKEND_BASE_NAME correctly\n" % (self.backendName,self.backendBaseName)
+            message = "Backend(s) not found, please set env CLIENT_HAPROXY_BACKEND_LIST correctly\n"
         if self.azLimiter == "true" and not fallbackBackendExists:
             result = False
             message += "Backend %s/%s not found, please set env CLIENT_HAPROXY_FALLBACK_BACKEND_NAME and CLIENT_HAPROXY_FALLBACK_BACKEND_BASE_NAME correctly" % (self.fallbackBackendName, self.fallbackBackendBaseName)
@@ -72,26 +86,25 @@ def setServer(self, server):
             self.sendHaproxyCommand(command)
 
     def prepareServer(self, server):
-        commands = []
         # If --az-limiter option is used
         if server.ASG not in self.ASG and self.azLimiter == "true":
-            commands.extend(self.__addServerInBackend(server, self.fallbackBackendName, self.fallbackBackendBaseName))
-        else:
-            commands.extend(self.__addServerInBackend(server, self.backendName, self.backendBaseName))
+            return self.__addServerInBackend(server, self.fallbackBackendName, self.fallbackBackendBaseName, self.backendServerPort)
+        # Read backend list
+        commands = []
+        for k,v in self.backendList.items():
+            commands.extend(self.__addServerInBackend(server, k, v["baseName"], v["serverPort"]))
             if self.optAllServersInFallback == "true":
-                commands.extend(self.__addServerInBackend(server, self.fallbackBackendName, self.fallbackBackendBaseName))
-        ## If server is disabled
+                commands.extend(self.__addServerInBackend(server, self.fallbackBackendName, self.fallbackBackendBaseName, self.backendServerPort))
         return commands
-
 
-    def __addServerInBackend(self, server, bckndName, bckndbsName):
+    def __addServerInBackend(self, server, bckndName, bckndbsName, bckndServerPort):
         commands = []
         if server.IPAddress == "none":
             commands.append(
                 "set server %s/%s state maint"
                 % (
-                    self.backendName,
-                    self.backendBaseName + str(server.backendServerID)
+                    bckndName,
+                    bckndbsName + str(server.backendServerID)
                 )
             )
         ## If server is enabled
@@ -102,7 +115,7 @@ def __addServerInBackend(self, server, bckndName, bckndbsName):
                     bckndName,
                     bckndbsName + str(server.backendServerID),
                     server.IPAddress,
-                    self.backendServerPort
+                    bckndServerPort,
                 )
             )
             commands.append(

src/common/configuration.py

@@ -18,7 +18,7 @@ def my_new(cls,*args,**kwds):
         cls.__new__ = staticmethod(my_new)
 
 ##
-# Get Consul Informations
+# Get Consul information
 ##
 class Configuration:
     __metaclass__ = SingletonMetaClass
@@ -36,7 +36,7 @@ def __init__(self):
         if "AWS_DEFAULT_REGION" not in self.__env:
             print("You must set a region AWS_DEFAULT_REGION")
             sys.exit(2)
-        ## AWS_DEFAULT_REGION must be in env var so boto3 could use it
+        ## AWS_DEFAULT_REGION env vars must set so that boto3 can use it
         os.environ["AWS_DEFAULT_REGION"] = self.__env["AWS_DEFAULT_REGION"]
 
     def get(self, key):

src/common/dynamodb.py

@@ -67,7 +67,7 @@ def listServers(self):
     def fillServer(self, item):
         s = ServerModel()
         s.backendServerID = int(item["BackendServerID"])
-        # if asg column isn't existing (problem occured when migrating from v2 to v3)
+        # if ASG column does not exist (problem occured when migrating from v2 to v3)
         if "ASG" in item:
             s.ASG = item["ASG"]
         s.IPAddress = item["IPAddress"]

src/common/logger.py

@@ -55,7 +55,7 @@ def info(self, message):
         self.logger.info("\033[0;0m%s\033[0;0m" % str(message))
 
     ##
-    # Error message
+    # Error messages
     ##
     def error(self, message):
         self.logger.error("\033[1;31m%s\033[0;0m" % str(message))

src/common/prometheus.py

@@ -22,7 +22,7 @@ def my_new(cls,*args,**kwds):
         cls.__new__ = staticmethod(my_new)
 
 ##
-# Export metrics to prometheus
+# Export metrics to Prometheus
 ##
 class Prometheus:
     __metaclass__ = SingletonMetaClass

src/server/server.py

@@ -74,7 +74,7 @@ def run(self):
                 self.logger.error("HAProxy backend size is lower than registered servers in dynamoDB. Please correct it by hand.")
                 sys.exit(2)
 
-            # Remove servers that does not exists anymore
+            # Remove servers that do not exist anymore
             self.removeUnexistingServers(dynamodbServers, sourceServers, oldDynamodbServers)
 
             # Do not keep track of already registered servers
@@ -92,7 +92,7 @@ def run(self):
                     Prometheus().serverWeightMetric(server)
 
                 match = False
-                #Update only modified servers
+                # Update only modified servers
                 for oldServer in oldDynamodbServers:
                     if server.equals(oldServer):
                         match = True
@@ -138,7 +138,7 @@ def removeDoublonServers(self, sourceServers : List[ServerModel], dynamodbServer
                 sourceServersToAdd.append(sServer)
         return sourceServersToAdd
 
-    # Remove servers that does not exists anymore
+    # Remove servers that do not exist anymore
     def removeUnexistingServers(self, dynamodbServers : List[ServerModel], sourceServers : List[ServerModel], oldDynamodbServers : List[ServerModel]):
         for oldServer in oldDynamodbServers:
             match = False