Move websocket skip logic into event_handler

awx/main/tasks.py

@@ -4,7 +4,7 @@
 # All Rights Reserved.
 
 # Python
-from collections import OrderedDict, namedtuple
+from collections import OrderedDict, namedtuple, deque
 import errno
 import functools
 import importlib
@@ -729,6 +729,10 @@ def with_path_cleanup(f):
     return _wrapped
 
 
+# TODO: move to CTiT settings if we ever get serious about this
+MAX_WEBSOCKET_EVENT_RATE = 30
+
+
 class BaseTask(object):
     model = None
     event_model = None
@@ -740,6 +744,7 @@ class BaseTask(object):
         self.host_map = {}
         self.guid = GuidMiddleware.get_guid()
         self.job_created = None
+        self.recent_event_timings = deque(maxlen=MAX_WEBSOCKET_EVENT_RATE)
 
     def update_model(self, pk, _attempt=0, **updates):
         """Reload the model instance from the database and update the
@@ -1151,6 +1156,27 @@ class BaseTask(object):
         if 'event_data' in event_data:
             event_data['event_data']['guid'] = self.guid
 
+        # To prevent overwhelming the broadcast queue, skip some websocket messages
+        cpu_time = time.time()
+        if self.recent_event_timings:
+            first_window_time = self.recent_event_timings[0]
+            inverse_effective_rate = cpu_time - first_window_time
+            # if last 30 events came in under 1 second ago
+            if inverse_effective_rate < 1.0:
+                if self.recent_event_timings[0] != self.recent_event_timings[-1]:
+                    logger.info('Too many events chief, not broadcasting because that would be crazy')
+                    # this is to smooth out jumpiness, we clear the events except for the last one
+                    # that will enforce that we wait a full second before starting again
+                    self.recent_event_timings.clear()
+                    self.recent_event_timings.append(first_window_time)
+                event_data['skip_websocket_message'] = True
+            else:
+                if self.recent_event_timings[0] == self.recent_event_timings[-1]:
+                    logger.info('Starting a window of event emission, will pause if I see too many')
+                self.recent_event_timings.append(cpu_time)
+        else:
+            self.recent_event_timings.append(cpu_time)
+
         event_data.setdefault(self.event_data_key, self.instance.id)
         self.dispatcher.dispatch(event_data)
         self.event_ct += 1