From 8350bb3371b9eb62a49e5370feb138f1603c9653 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 21 Feb 2020 13:51:47 -0500 Subject: [PATCH] robust broadcast websocket error hanndling --- awx/main/wsbroadcast.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index 7aef604f52..f7171a8627 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -2,11 +2,13 @@ import os import json import logging -import aiohttp import asyncio import datetime import sys +import aiohttp +from aiohttp import client_exceptions + from channels_redis.core import RedisChannelLayer from channels.layers import get_channel_layer @@ -107,12 +109,18 @@ class WebsocketTask(): logger.warn(f"{self.name} connection to {self.remote_host} cancelled") self.stats.record_connection_lost() raise + except client_exceptions.ClientConnectorError as e: + logger.warn(f"Failed to connect to {self.remote_host}: '{e}'. Reconnecting ...") + self.stats.record_connection_lost() + self.start(attempt=attempt+1) + except asyncio.TimeoutError: + logger.warn(f"Timeout while trying to connect to {self.remote_host}. Reconnecting ...") + self.stats.record_connection_lost() + self.start(attempt=attempt+1) except Exception as e: # Early on, this is our canary. I'm not sure what exceptions we can really encounter. - # Does aiohttp throws an exception if a disconnect happens? - logger.warn(f"Websocket broadcast client exception {str(e)}") + logger.warn(f"Websocket broadcast client exception {type(e)} {e}") self.stats.record_connection_lost() - # Reconnect self.start(attempt=attempt+1) def start(self, attempt=0):