From bac124004fe80baf5462c80d39de2162d9937a32 Mon Sep 17 00:00:00 2001 From: Rick Elrod Date: Wed, 24 May 2023 13:27:55 -0500 Subject: [PATCH] Rename heartbeet daemon to ws_heartbeat (#14041) Signed-off-by: Rick Elrod --- Makefile | 4 +- .../{run_heartbeet.py => run_ws_heartbeat.py} | 37 ++----------------- awx/main/wsrelay.py | 14 +++---- awx/settings/defaults.py | 4 +- docs/websockets.md | 8 ++-- .../templates/supervisor_web.conf.j2 | 8 ++-- tools/docker-compose/supervisor.conf | 6 +-- 7 files changed, 26 insertions(+), 55 deletions(-) rename awx/main/management/commands/{run_heartbeet.py => run_ws_heartbeat.py} (50%) diff --git a/Makefile b/Makefile index 68e519bd87..33e03fb561 100644 --- a/Makefile +++ b/Makefile @@ -267,11 +267,11 @@ run-wsrelay: $(PYTHON) manage.py run_wsrelay ## Start the heartbeat process in background in development environment. -run-heartbeet: +run-ws-heartbeat: @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/awx/bin/activate; \ fi; \ - $(PYTHON) manage.py run_heartbeet + $(PYTHON) manage.py run_ws_heartbeat reports: mkdir -p $@ diff --git a/awx/main/management/commands/run_heartbeet.py b/awx/main/management/commands/run_ws_heartbeat.py similarity index 50% rename from awx/main/management/commands/run_heartbeet.py rename to awx/main/management/commands/run_ws_heartbeat.py index cfe86bc25c..e7f08b1d89 100644 --- a/awx/main/management/commands/run_heartbeet.py +++ b/awx/main/management/commands/run_ws_heartbeat.py @@ -10,39 +10,11 @@ from django.conf import settings from awx.main.dispatch import pg_bus_conn -logger = logging.getLogger('awx.main.commands.run_heartbeet') +logger = logging.getLogger('awx.main.commands.run_ws_heartbeat') class Command(BaseCommand): - help = 'Launch the web server beacon (heartbeet)' - - def print_banner(self): - heartbeet = r""" - ********** ********** - ************* ************* -***************************** - ***********HEART*********** - ************************* - ******************* - *************** _._ - *********** /`._ `'. __ - ******* \ .\| \ _'` `) - *** (``_) \| ).'` /`- / - * `\ `;\_ `\\//`-'` / - \ `'.'.| / __/` - `'--v_|/`'` - __||-._ - /'` `-`` `'\\ - / .'` ) - \ BEET ' ) - \. / - '. /'` - `) | - // - '(. - `\`. - ``""" - print(heartbeet) + help = 'Launch the web server beacon (ws_heartbeat)' def construct_payload(self, action='online'): payload = { @@ -54,18 +26,17 @@ class Command(BaseCommand): def notify_listener_and_exit(self, *args): with pg_bus_conn(new_connection=False) as conn: - conn.notify('web_heartbeet', self.construct_payload(action='offline')) + conn.notify('web_ws_heartbeat', self.construct_payload(action='offline')) sys.exit(0) def do_hearbeat_loop(self): with pg_bus_conn(new_connection=True) as conn: while True: logger.debug('Sending heartbeat') - conn.notify('web_heartbeet', self.construct_payload()) + conn.notify('web_ws_heartbeat', self.construct_payload()) time.sleep(settings.BROADCAST_WEBSOCKET_BEACON_FROM_WEB_RATE_SECONDS) def handle(self, *arg, **options): - self.print_banner() signal.signal(signal.SIGTERM, self.notify_listener_and_exit) signal.signal(signal.SIGINT, self.notify_listener_and_exit) diff --git a/awx/main/wsrelay.py b/awx/main/wsrelay.py index 65ccf3537b..9ce5d9e22d 100644 --- a/awx/main/wsrelay.py +++ b/awx/main/wsrelay.py @@ -209,15 +209,15 @@ class WebSocketRelayManager(object): # hostname -> ip self.known_hosts: Dict[str, str] = dict() - async def on_heartbeet(self, conn, pid, channel, payload): + async def on_ws_heartbeat(self, conn, pid, channel, payload): try: - if not payload or channel != "web_heartbeet": + if not payload or channel != "web_ws_heartbeat": return try: payload = json.loads(payload) except json.JSONDecodeError: - logmsg = "Failed to decode message from pg_notify channel `web_heartbeet`" + logmsg = "Failed to decode message from pg_notify channel `web_ws_heartbeat`" if logger.isEnabledFor(logging.DEBUG): logmsg = "{} {}".format(logmsg, payload) logger.warning(logmsg) @@ -235,7 +235,7 @@ class WebSocketRelayManager(object): # If we don't get an IP, just try the hostname, maybe it resolves ip = hostname if ip is None: - logger.warning(f"Received invalid online heartbeet, missing hostname and ip: {payload}") + logger.warning(f"Received invalid online ws_heartbeat, missing hostname and ip: {payload}") return self.known_hosts[hostname] = ip logger.debug(f"Web host {hostname} ({ip}) online heartbeat received.") @@ -246,14 +246,14 @@ class WebSocketRelayManager(object): # If we don't get an IP, just try the hostname, maybe it resolves ip = hostname if ip is None: - logger.warning(f"Received invalid offline heartbeet, missing hostname and ip: {payload}") + logger.warning(f"Received invalid offline ws_heartbeat, missing hostname and ip: {payload}") return self.cleanup_offline_host(ip) logger.debug(f"Web host {hostname} ({ip}) offline heartbeat received.") except Exception as e: # This catch-all is the same as the one above. asyncio will eat the exception # but we want to know about it. - logger.exception(f"on_heartbeet exception: {e}") + logger.exception(f"on_ws_heartbeat exception: {e}") def cleanup_offline_host(self, hostname): """ @@ -291,7 +291,7 @@ class WebSocketRelayManager(object): # We cannot include these because asyncpg doesn't allow all the options that psycopg does. # **database_conf.get("OPTIONS", {}), ) - await async_conn.add_listener("web_heartbeet", self.on_heartbeet) + await async_conn.add_listener("web_ws_heartbeat", self.on_ws_heartbeat) # Establishes a websocket connection to /websocket/relay on all API servers while True: diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index c55fa12e97..abaad16223 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -856,7 +856,7 @@ LOGGING = { 'awx.main.consumers': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'INFO'}, 'awx.main.rsyslog_configurer': {'handlers': ['rsyslog_configurer']}, 'awx.main.cache_clear': {'handlers': ['cache_clear']}, - 'awx.main.heartbeet': {'handlers': ['heartbeet']}, + 'awx.main.ws_heartbeat': {'handlers': ['ws_heartbeat']}, 'awx.main.wsrelay': {'handlers': ['wsrelay']}, 'awx.main.commands.inventory_import': {'handlers': ['inventory_import'], 'propagate': False}, 'awx.main.tasks': {'handlers': ['task_system', 'external_logger', 'console'], 'propagate': False}, @@ -890,7 +890,7 @@ handler_config = { 'job_lifecycle': {'filename': 'job_lifecycle.log', 'formatter': 'job_lifecycle'}, 'rsyslog_configurer': {'filename': 'rsyslog_configurer.log'}, 'cache_clear': {'filename': 'cache_clear.log'}, - 'heartbeet': {'filename': 'heartbeet.log'}, + 'ws_heartbeat': {'filename': 'ws_heartbeat.log'}, } # If running on a VM, we log to files. When running in a container, we log to stdout. diff --git a/docs/websockets.md b/docs/websockets.md index 5c2e57a6a9..24128260b7 100644 --- a/docs/websockets.md +++ b/docs/websockets.md @@ -44,7 +44,7 @@ The notable modules for this component are: endpoint. This is a daemon. It formerly ran in each web container, but now runs in each task container instead. -* `awx/main/management/commands/run_heartbeet.py` - discussed below, used to +* `awx/main/management/commands/run_ws_heartbeat.py` - discussed below, used to send a heartbeat payload to pg_notify every few seconds, so that all task pods running `wsrelay.py` (above) know about each web pod. @@ -103,7 +103,7 @@ that care about them. ### The Heartbeet -There is also a "heartbeet" system (a play on "heartbeat"), that goes along with +There is also a "ws_heartbeat" system, that goes along with the above. Remember that `wsrelay` lives in each task pod, and there could be an arbitrary number of web and task pods (independent of each other). Because of this, `wsrelay` (on all task pods) needs to know which web pods are up and need @@ -111,7 +111,7 @@ to be connected to (on their "relay" endpoints). To accomplish this, we use pg_notify, since web and task pods are all able to connect to the database and we are safely able to use it as a central communication point. -In each web container, there is a process, `run_heartbeet.py` which will send +In each web container, there is a process, `run_ws_heartbeat.py` which will send out a heartbeat payload to pg_notify, every `settings.BROADCAST_WEBSOCKET_BEACON_FROM_WEB_RATE_SECONDS` seconds. This is done in a broadcast fashion to a specific pg_notify channel, and each `wsrelay` @@ -120,7 +120,7 @@ these messages. When `wsrelay` sees this heartbeat packet, it checks to see if the web node is known already. If not, it creates a connection to it, and adds it to its list of nodes to relay websocket messages to. -It can also handle web nodes going offline. If `run_heartbeet.py` detects +It can also handle web nodes going offline. If `run_ws_heartbeat.py` detects SIGTERM or SIGINT, it will send an "offline" heartbeat packet, and `wsrelay` will work to *remove* the web node from its list of active connections. diff --git a/tools/ansible/roles/dockerfile/templates/supervisor_web.conf.j2 b/tools/ansible/roles/dockerfile/templates/supervisor_web.conf.j2 index fcd32efc0d..6112b6abda 100644 --- a/tools/ansible/roles/dockerfile/templates/supervisor_web.conf.j2 +++ b/tools/ansible/roles/dockerfile/templates/supervisor_web.conf.j2 @@ -55,12 +55,12 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -[program:heartbeet] +[program:ws-heartbeat] {% if kube_dev | bool %} -command = make run-heartbeet +command = make run-ws-heartbeat directory = /awx_devel {% else %} -command = awx-manage run_heartbeet +command = awx-manage run_ws_heartbeat directory = /var/lib/awx {% endif %} autorestart = true @@ -103,7 +103,7 @@ stderr_logfile_maxbytes=0 {% endif %} [group:tower-processes] -programs=nginx,uwsgi,daphne,awx-cache-clear,heartbeet +programs=nginx,uwsgi,daphne,awx-cache-clear,ws-heartbeat priority=5 [eventlistener:superwatcher] diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf index 704e4d2dcd..2b3deb2eab 100644 --- a/tools/docker-compose/supervisor.conf +++ b/tools/docker-compose/supervisor.conf @@ -28,8 +28,8 @@ killasgroup=true stdout_events_enabled = true stderr_events_enabled = true -[program:awx-heartbeet] -command = awx-manage run_heartbeet +[program:awx-ws-heartbeat] +command = awx-manage run_ws_heartbeat autorestart = true autorestart = true stopasgroup=true @@ -101,7 +101,7 @@ stdout_events_enabled = true stderr_events_enabled = true [group:tower-processes] -programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx,awx-wsrelay,awx-rsyslogd,awx-heartbeet,awx-rsyslog-configurer,awx-cache-clear +programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx,awx-wsrelay,awx-rsyslogd,awx-ws-heartbeat,awx-rsyslog-configurer,awx-cache-clear priority=5 [program:awx-autoreload]