From 9c20b9412af61094d37a6e2d0cb6c9fc7a5d2ad3 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Wed, 13 May 2020 08:30:39 -0400 Subject: [PATCH 1/2] delete and re-add host when ip address changes * The websocket backplane interconnect is done via ip address for Kubernetes and OpenShift. On init run_wsbroadcast reads all Instances from the DB and makes a decision to use the ip address or the hostname based, with preference given to the ip address if defined. For Kubernetes and OpenShift the nodes can load the Instance before the ip_address is set. This would cause the connection to be tried by hostname rather than ip address. This changeset ensures that an ip address set after an Instance record is created will be detected and used. --- awx/main/wsbroadcast.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index 243981a885..05ab30602f 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -162,6 +162,13 @@ class BroadcastWebsocketManager(object): deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts) new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts) + remote_addresses = {k: v.remote_host for k, v in self.broadcast_tasks.items()} + for hostname, address in known_hosts.items(): + if hostname in self.broadcast_tasks and \ + address != remote_addresses[hostname]: + deleted_remote_hosts.add(hostname) + new_remote_hosts.add(hostname) + if deleted_remote_hosts: logger.warn(f"Removing {deleted_remote_hosts} from websocket broadcast list") if new_remote_hosts: From 2f7ba75ae4c4a054dde1cae098fc609aae9b6946 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 12 May 2020 13:23:08 -0400 Subject: [PATCH 2/2] track stats by hostname not remote host/ip * broadcast websockets have stats tracked (i.e. connection status, number of messages total, messages per minute, etc). Previous to this change, stats were tracked by ip address, if it was defined on the instance, XOR hostname. This changeset tracks stats by hostname. --- awx/main/wsbroadcast.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index 05ab30602f..a97baf45f4 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -37,7 +37,7 @@ def get_broadcast_hosts(): .order_by('hostname') \ .values('hostname', 'ip_address') \ .distinct() - return [i['ip_address'] or i['hostname'] for i in instances] + return {i['hostname']: i['ip_address'] or i['hostname'] for i in instances} def get_local_host(): @@ -149,15 +149,22 @@ class BroadcastWebsocketTask(WebsocketTask): class BroadcastWebsocketManager(object): def __init__(self): self.event_loop = asyncio.get_event_loop() + ''' + { + 'hostname1': BroadcastWebsocketTask(), + 'hostname2': BroadcastWebsocketTask(), + 'hostname3': BroadcastWebsocketTask(), + } + ''' self.broadcast_tasks = dict() - # parallel dict to broadcast_tasks that tracks stats self.local_hostname = get_local_host() self.stats_mgr = BroadcastWebsocketStatsManager(self.event_loop, self.local_hostname) async def run_per_host_websocket(self): while True: - future_remote_hosts = get_broadcast_hosts() + known_hosts = get_broadcast_hosts() + future_remote_hosts = known_hosts.keys() current_remote_hosts = self.broadcast_tasks.keys() deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts) new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts) @@ -184,7 +191,7 @@ class BroadcastWebsocketManager(object): broadcast_task = BroadcastWebsocketTask(name=self.local_hostname, event_loop=self.event_loop, stats=stats, - remote_host=h) + remote_host=known_hosts[h]) broadcast_task.start() self.broadcast_tasks[h] = broadcast_task