Fixing wsrelay connection loop (#14692)

* Fixing wsrelay connection loop

* The loop was being interrupted when reaching the return statements, causing a race condition that would make nodes remain disconnected from their websockets
* Added log messages for the previous return state to improve the logging from this state.

* Added logging for malformed payload

* Update awx/main/wsrelay.py

Co-authored-by: Rick Elrod <rick@elrod.me>

* Moved logmsg outside condition

---------

Co-authored-by: Lucas Benedito <lbenedit@redhat.com>
Co-authored-by: Rick Elrod <rick@elrod.me>

awx/main/wsrelay.py

@@ -227,21 +227,24 @@ async def on_ws_heartbeat(self, conn):
                 continue
             try:
                 if not notif.payload or notif.channel != "web_ws_heartbeat":
-                    return
+                    logger.warning(f"Unexpected channel or missing payload. {notif.channel}, {notif.payload}")
+                    continue
 
                 try:
                     payload = json.loads(notif.payload)
                 except json.JSONDecodeError:
                     logmsg = "Failed to decode message from pg_notify channel `web_ws_heartbeat`"
                     if logger.isEnabledFor(logging.DEBUG):
                         logmsg = "{} {}".format(logmsg, payload)
-                        logger.warning(logmsg)
-                    return
+                    logger.warning(logmsg)
+                    continue
 
                 # Skip if the message comes from the same host we are running on
                 # In this case, we'll be sharing a redis, no need to relay.
                 if payload.get("hostname") == self.local_hostname:
-                    return
+                    hostname = payload.get("hostname")
+                    logger.debug("Received a heartbeat request for {hostname}. Skipping as we use redis for local host.")
+                    continue
 
                 action = payload.get("action")
 
@@ -250,7 +253,7 @@ async def on_ws_heartbeat(self, conn):
                     ip = payload.get("ip") or hostname  # try back to hostname if ip isn't supplied
                     if ip is None:
                         logger.warning(f"Received invalid {action} ws_heartbeat, missing hostname and ip: {payload}")
-                        return
+                        continue
                     logger.debug(f"Web host {hostname} ({ip}) {action} heartbeat received.")
 
                 if action == "online":