From 45819f6b9a7f071ed52a2fa4ce59540789533c07 Mon Sep 17 00:00:00 2001
From: Ryan Petrello <rpetrell@redhat.com>
Date: Tue, 28 Feb 2017 09:34:44 -0500
Subject: [PATCH] requeue websocket messages that don't (yet) have an
 established user

There's a race between our `ws_connect` and `ws_receive` methods;
it's possible to fall into a scenario where we're handling a legitimate
message *before* django-channels is able to persist the `user_id` into
the channel session.  This results in a scenario where a user can open
a browser tab and never receive new websocket messages.  In this
scenario, we should just toss the message back into the queue and try
again later (up to a reasonable limit of retries).
---
 awx/main/consumers.py    | 13 +++++++++++--
 awx/settings/defaults.py |  5 +++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/awx/main/consumers.py b/awx/main/consumers.py
index c42f16ef21..ff55507939 100644
--- a/awx/main/consumers.py
+++ b/awx/main/consumers.py
@@ -2,10 +2,11 @@ import json
 import logging
 import urllib
 
-from channels import Group
+from channels import Group, channel_layers
 from channels.sessions import channel_session
 from channels.handler import AsgiRequest
 
+from django.conf import settings
 from django.core.serializers.json import DjangoJSONEncoder
 
 from django.contrib.auth.models import User
@@ -49,11 +50,19 @@ def ws_disconnect(message):
 @channel_session
 def ws_receive(message):
     from awx.main.access import consumer_access
+    channel_layer_settings = channel_layers.configs[message.channel_layer.alias]
+    max_retries = channel_layer_settings.get('RECEIVE_MAX_RETRY', settings.CHANNEL_LAYER_RECEIVE_MAX_RETRY)
 
     user_id = message.channel_session.get('user_id', None)
     if user_id is None:
-        logger.error("No valid user found for websocket.")
+        retries = message.content.get('connect_retries', 0) + 1
+        message.content['connect_retries'] = retries
         message.reply_channel.send({"text": json.dumps({"error": "no valid user"})})
+        retries_left = max_retries - retries
+        if retries_left > 0:
+            message.channel_layer.send(message.channel.name, message.content)
+        else:
+            logger.error("No valid user found for websocket.")
         return None
 
     user = User.objects.get(pk=user_id)
diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
index 090e939914..0901961fa5 100644
--- a/awx/settings/defaults.py
+++ b/awx/settings/defaults.py
@@ -866,6 +866,11 @@ TOWER_SETTINGS_MANIFEST = {}
 
 LOG_AGGREGATOR_ENABLED = False
 
+# The number of retry attempts for websocket session establishment
+# If you're encountering issues establishing websockets in clustered Tower,
+# raising this value can help
+CHANNEL_LAYER_RECEIVE_MAX_RETRY = 10
+
 # Logging configuration.
 LOGGING = {
     'version': 1,