requeue websocket messages that don't (yet) have an established user

There's a race between our `ws_connect` and `ws_receive` methods;
it's possible to fall into a scenario where we're handling a legitimate
message *before* django-channels is able to persist the `user_id` into
the channel session.  This results in a scenario where a user can open
a browser tab and never receive new websocket messages.  In this
scenario, we should just toss the message back into the queue and try
again later (up to a reasonable limit of retries).

awx/main/consumers.py

@@ -2,10 +2,11 @@ import json
 import logging
 import urllib
 
-from channels import Group
+from channels import Group, channel_layers
 from channels.sessions import channel_session
 from channels.handler import AsgiRequest
 
+from django.conf import settings
 from django.core.serializers.json import DjangoJSONEncoder
 
 from django.contrib.auth.models import User
@@ -49,11 +50,19 @@ def ws_disconnect(message):
 @channel_session
 def ws_receive(message):
     from awx.main.access import consumer_access
+    channel_layer_settings = channel_layers.configs[message.channel_layer.alias]
+    max_retries = channel_layer_settings.get('RECEIVE_MAX_RETRY', settings.CHANNEL_LAYER_RECEIVE_MAX_RETRY)
 
     user_id = message.channel_session.get('user_id', None)
     if user_id is None:
-        logger.error("No valid user found for websocket.")
+        retries = message.content.get('connect_retries', 0) + 1
+        message.content['connect_retries'] = retries
         message.reply_channel.send({"text": json.dumps({"error": "no valid user"})})
+        retries_left = max_retries - retries
+        if retries_left > 0:
+            message.channel_layer.send(message.channel.name, message.content)
+        else:
+            logger.error("No valid user found for websocket.")
         return None
 
     user = User.objects.get(pk=user_id)

awx/settings/defaults.py

@@ -866,6 +866,11 @@ TOWER_SETTINGS_MANIFEST = {}
 
 LOG_AGGREGATOR_ENABLED = False
 
+# The number of retry attempts for websocket session establishment
+# If you're encountering issues establishing websockets in clustered Tower,
+# raising this value can help
+CHANNEL_LAYER_RECEIVE_MAX_RETRY = 10
+
 # Logging configuration.
 LOGGING = {
     'version': 1,