Merge pull request #6034 from chrismeyersfsu/pg2_no_pubsub

Replace rabbitmq with redis

Makefile

@@ -265,28 +265,6 @@ migrate:
 dbchange:
 	$(MANAGEMENT_COMMAND) makemigrations
 
-server_noattach:
-	tmux new-session -d -s awx 'exec make uwsgi'
-	tmux rename-window 'AWX'
-	tmux select-window -t awx:0
-	tmux split-window -v 'exec make dispatcher'
-	tmux new-window 'exec make daphne'
-	tmux select-window -t awx:1
-	tmux rename-window 'WebSockets'
-	tmux split-window -h 'exec make runworker'
-	tmux split-window -v 'exec make nginx'
-	tmux new-window 'exec make receiver'
-	tmux select-window -t awx:2
-	tmux rename-window 'Extra Services'
-	tmux select-window -t awx:0
-
-server: server_noattach
-	tmux -2 attach-session -t awx
-
-# Use with iterm2's native tmux protocol support
-servercc: server_noattach
-	tmux -2 -CC attach-session -t awx
-
 supervisor:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
@@ -311,18 +289,11 @@ daphne:
 	fi; \
 	daphne -b 127.0.0.1 -p 8051 awx.asgi:channel_layer
 
-runworker:
+wsbroadcast:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	$(PYTHON) manage.py runworker --only-channels websocket.*
-
-# Run the built-in development webserver (by default on http://localhost:8013).
-runserver:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	$(PYTHON) manage.py runserver
+	$(PYTHON) manage.py run_wsbroadcast
 
 # Run to start the background task dispatcher for development.
 dispatcher:

awx/api/generics.py

@@ -5,10 +5,12 @@
 import inspect
 import logging
 import time
+import uuid
 import urllib.parse
 
 # Django
 from django.conf import settings
+from django.core.cache import cache
 from django.db import connection
 from django.db.models.fields import FieldDoesNotExist
 from django.db.models.fields.related import OneToOneRel
@@ -973,6 +975,11 @@ class CopyAPIView(GenericAPIView):
         if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role.members.all():
             new_obj.admin_role.members.add(request.user)
         if sub_objs:
+            # store the copied object dict into memcached, because it's
+            # often too large for postgres' notification bus
+            # (which has a default maximum message size of 8k)
+            key = 'deep-copy-{}'.format(str(uuid.uuid4()))
+            cache.set(key, sub_objs, timeout=3600)
             permission_check_func = None
             if hasattr(type(self), 'deep_copy_permission_check_func'):
                 permission_check_func = (
@@ -980,7 +987,7 @@ class CopyAPIView(GenericAPIView):
                 )
             trigger_delayed_deep_copy(
                 self.model.__module__, self.model.__name__,
-                obj.pk, new_obj.pk, request.user.pk, sub_objs,
+                obj.pk, new_obj.pk, request.user.pk, key,
                 permission_check_func=permission_check_func
             )
         serializer = self._get_copy_return_serializer(new_obj)

awx/api/urls/urls.py

@@ -34,7 +34,9 @@ from awx.api.views import (
     OAuth2ApplicationDetail,
 )
 
-from awx.api.views.metrics import MetricsView
+from awx.api.views.metrics import (
+    MetricsView,
+)
 
 from .organization import urls as organization_urls
 from .user import urls as user_urls

awx/api/views/metrics.py

@@ -39,3 +39,4 @@ class MetricsView(APIView):
         if (request.user.is_superuser or request.user.is_system_auditor):
             return Response(metrics().decode('UTF-8'))
         raise PermissionDenied()
+

awx/asgi.py

@@ -2,14 +2,15 @@
 # All Rights Reserved.
 import os
 import logging
+import django
 from awx import __version__ as tower_version
 
 # Prepare the AWX environment.
 from awx import prepare_env, MODE
 prepare_env() # NOQA
 
-from django.core.wsgi import get_wsgi_application  # NOQA
-from channels.asgi import get_channel_layer
+from channels.routing import get_default_application
+
 
 """
 ASGI config for AWX project.
@@ -32,6 +33,5 @@ if MODE == 'production':
 
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "awx.settings")
-
-
-channel_layer = get_channel_layer()
+django.setup()
+channel_layer = get_default_application()

awx/main/analytics/broadcast_websocket.py

@@ -0,0 +1,166 @@
+import datetime
+import asyncio
+import logging
+import aioredis
+import redis
+import re
+
+from prometheus_client import (
+    generate_latest,
+    Gauge,
+    Counter,
+    Enum,
+    CollectorRegistry,
+)
+
+from django.conf import settings
+
+
+BROADCAST_WEBSOCKET_REDIS_KEY_NAME = 'broadcast_websocket_stats'
+
+
+logger = logging.getLogger('awx.main.analytics.broadcast_websocket')
+
+
+def dt_to_seconds(dt):
+    return int((dt - datetime.datetime(1970,1,1)).total_seconds())
+
+
+def now_seconds():
+    return dt_to_seconds(datetime.datetime.now())
+
+
+# Second granularity; Per-minute
+class FixedSlidingWindow():
+    def __init__(self, start_time=None):
+        self.buckets = dict()
+        self.start_time = start_time or now_seconds()
+
+    def cleanup(self, now_bucket=None):
+        now_bucket = now_bucket or now_seconds()
+        if self.start_time + 60 <= now_bucket:
+            self.start_time = now_bucket + 60 + 1
+
+            # Delete old entries
+            for k in list(self.buckets.keys()):
+                if k < self.start_time:
+                    del self.buckets[k]
+
+    def record(self, ts=None):
+        ts = ts or datetime.datetime.now()
+        now_bucket = int((ts - datetime.datetime(1970,1,1)).total_seconds())
+
+        val = self.buckets.get(now_bucket, 0)
+        self.buckets[now_bucket] = val + 1
+
+        self.cleanup(now_bucket)
+
+    def render(self):
+        self.cleanup()
+        return sum(self.buckets.values()) or 0
+
+
+class BroadcastWebsocketStatsManager():
+    def __init__(self, event_loop, local_hostname):
+        self._local_hostname = local_hostname
+
+        self._event_loop = event_loop
+        self._stats = dict()
+        self._redis_key = BROADCAST_WEBSOCKET_REDIS_KEY_NAME
+
+    def new_remote_host_stats(self, remote_hostname):
+        self._stats[remote_hostname] = BroadcastWebsocketStats(self._local_hostname,
+                                                               remote_hostname)
+        return self._stats[remote_hostname]
+
+    def delete_remote_host_stats(self, remote_hostname):
+        del self._stats[remote_hostname]
+
+    async def run_loop(self):
+        try:
+            redis_conn = await aioredis.create_redis_pool(settings.BROKER_URL)
+            while True:
+                stats_data_str = ''.join(stat.serialize() for stat in self._stats.values())
+                await redis_conn.set(self._redis_key, stats_data_str)
+
+                await asyncio.sleep(settings.BROADCAST_WEBSOCKET_STATS_POLL_RATE_SECONDS)
+        except Exception as e:
+            logger.warn(e)
+            await asyncio.sleep(settings.BROADCAST_WEBSOCKET_STATS_POLL_RATE_SECONDS)
+            self.start()
+
+    def start(self):
+        self.async_task = self._event_loop.create_task(self.run_loop())
+        return self.async_task
+
+    @classmethod
+    def get_stats_sync(cls):
+        '''
+        Stringified verion of all the stats
+        '''
+        redis_conn = redis.Redis.from_url(settings.BROKER_URL)
+        return redis_conn.get(BROADCAST_WEBSOCKET_REDIS_KEY_NAME)
+
+
+class BroadcastWebsocketStats():
+    def __init__(self, local_hostname, remote_hostname):
+        self._local_hostname = local_hostname
+        self._remote_hostname = remote_hostname
+        self._registry = CollectorRegistry()
+
+        # TODO: More robust replacement
+        self.name = self.safe_name(self._local_hostname)
+        self.remote_name = self.safe_name(self._remote_hostname)
+
+        self._messages_received_total = Counter(f'awx_{self.remote_name}_messages_received_total',
+                                                'Number of messages received, to be forwarded, by the broadcast websocket system',
+                                                registry=self._registry)
+        self._messages_received = Gauge(f'awx_{self.remote_name}_messages_received',
+                                        'Number forwarded messages received by the broadcast websocket system, for the duration of the current connection',
+                                        registry=self._registry)
+        self._connection = Enum(f'awx_{self.remote_name}_connection',
+                                'Websocket broadcast connection',
+                                states=['disconnected', 'connected'],
+                                registry=self._registry)
+        self._connection_start = Gauge(f'awx_{self.remote_name}_connection_start',
+                                       'Time the connection was established',
+                                       registry=self._registry)
+
+        self._messages_received_per_minute = Gauge(f'awx_{self.remote_name}_messages_received_per_minute',
+                                                   'Messages received per minute',
+                                                   registry=self._registry)
+        self._internal_messages_received_per_minute = FixedSlidingWindow()
+
+    def safe_name(self, s):
+        # Replace all non alpha-numeric characters with _
+        return re.sub('[^0-9a-zA-Z]+', '_', s)
+
+    def unregister(self):
+        self._registry.unregister(f'awx_{self.remote_name}_messages_received')
+        self._registry.unregister(f'awx_{self.remote_name}_connection')
+
+    def record_message_received(self):
+        self._internal_messages_received_per_minute.record()
+        self._messages_received.inc()
+        self._messages_received_total.inc()
+
+    def record_connection_established(self):
+        self._connection.state('connected')
+        self._connection_start.set_to_current_time()
+        self._messages_received.set(0)
+
+    def record_connection_lost(self):
+        self._connection.state('disconnected')
+
+    def get_connection_duration(self):
+        return (datetime.datetime.now() - self._connection_established_ts).total_seconds()
+
+    def render(self):
+        msgs_per_min = self._internal_messages_received_per_minute.render()
+        self._messages_received_per_minute.set(msgs_per_min)
+
+    def serialize(self):
+        self.render()
+
+        registry_data = generate_latest(self._registry).decode('UTF-8')
+        return registry_data

awx/main/consumers.py

@@ -1,97 +1,266 @@
 import json
 import logging
+import datetime
+import hmac
+import asyncio
 
-from channels import Group
-from channels.auth import channel_session_user_from_http, channel_session_user
-
-from django.utils.encoding import smart_str
-from django.http.cookie import parse_cookie
 from django.core.serializers.json import DjangoJSONEncoder
+from django.conf import settings
+from django.utils.encoding import force_bytes
+from django.contrib.auth.models import User
+
+from channels.generic.websocket import AsyncJsonWebsocketConsumer
+from channels.layers import get_channel_layer
+from channels.db import database_sync_to_async
 
 
 logger = logging.getLogger('awx.main.consumers')
 XRF_KEY = '_auth_user_xrf'
 
 
-def discard_groups(message):
-    if 'groups' in message.channel_session:
-        for group in message.channel_session['groups']:
-            Group(group).discard(message.reply_channel)
+class WebsocketSecretAuthHelper:
+    """
+    Middlewareish for websockets to verify node websocket broadcast interconnect.
+
+    Note: The "ish" is due to the channels routing interface. Routing occurs 
+    _after_ authentication; making it hard to apply this auth to _only_ a subset of
+    websocket endpoints.
+    """
+
+    @classmethod
+    def construct_secret(cls):
+        nonce_serialized = "{}".format(int((datetime.datetime.utcnow() - datetime.datetime.fromtimestamp(0)).total_seconds()))
+        payload_dict = {
+            'secret': settings.BROADCAST_WEBSOCKET_SECRET,
+            'nonce': nonce_serialized
+        }
+        payload_serialized = json.dumps(payload_dict)
+
+        secret_serialized = hmac.new(force_bytes(settings.BROADCAST_WEBSOCKET_SECRET),
+                                     msg=force_bytes(payload_serialized),
+                                     digestmod='sha256').hexdigest()
+
+        return 'HMAC-SHA256 {}:{}'.format(nonce_serialized, secret_serialized)
 
 
-@channel_session_user_from_http
-def ws_connect(message):
-    headers = dict(message.content.get('headers', ''))
-    message.reply_channel.send({"accept": True})
-    message.content['method'] = 'FAKE'
-    if message.user.is_authenticated:
-        message.reply_channel.send(
-            {"text": json.dumps({"accept": True, "user": message.user.id})}
-        )
-        # store the valid CSRF token from the cookie so we can compare it later
-        # on ws_receive
-        cookie_token = parse_cookie(
-            smart_str(headers.get(b'cookie'))
-        ).get('csrftoken')
-        if cookie_token:
-            message.channel_session[XRF_KEY] = cookie_token
-    else:
-        logger.error("Request user is not authenticated to use websocket.")
-        message.reply_channel.send({"close": True})
-    return None
+    @classmethod
+    def verify_secret(cls, s, nonce_tolerance=300):
+        try:
+            (prefix, payload) = s.split(' ')
+            if prefix != 'HMAC-SHA256':
+                raise ValueError('Unsupported encryption algorithm')
+            (nonce_parsed, secret_parsed) = payload.split(':')
+        except Exception:
+            raise ValueError("Failed to parse secret")
+
+        try:
+            payload_expected = {
+                'secret': settings.BROADCAST_WEBSOCKET_SECRET,
+                'nonce': nonce_parsed,
+            }
+            payload_serialized = json.dumps(payload_expected)
+        except Exception:
+            raise ValueError("Failed to create hash to compare to secret.")
+
+        secret_serialized = hmac.new(force_bytes(settings.BROADCAST_WEBSOCKET_SECRET),
+                                     msg=force_bytes(payload_serialized),
+                                     digestmod='sha256').hexdigest()
+
+        if secret_serialized != secret_parsed:
+            raise ValueError("Invalid secret")
+
+        # Avoid timing attack and check the nonce after all the heavy lifting
+        now = datetime.datetime.utcnow()
+        nonce_parsed = datetime.datetime.fromtimestamp(int(nonce_parsed))
+        if (now - nonce_parsed).total_seconds() > nonce_tolerance:
+            raise ValueError("Potential replay attack or machine(s) time out of sync.")
+
+        return True
+
+    @classmethod
+    def is_authorized(cls, scope):
+        secret = ''
+        for k, v in scope['headers']:
+            if k.decode("utf-8") == 'secret':
+                secret = v.decode("utf-8")
+                break
+        WebsocketSecretAuthHelper.verify_secret(secret)
 
 
-@channel_session_user
-def ws_disconnect(message):
-    discard_groups(message)
+class BroadcastConsumer(AsyncJsonWebsocketConsumer):
+
+    async def connect(self):
+        try:
+            WebsocketSecretAuthHelper.is_authorized(self.scope)
+        except Exception:
+            # TODO: log ip of connected client
+            logger.warn("Broadcast client failed to authorize.")
+            await self.close()
+            return
+
+        # TODO: log ip of connected client
+        logger.info(f"Broadcast client connected.")
+        await self.accept()
+        await self.channel_layer.group_add(settings.BROADCAST_WEBSOCKET_GROUP_NAME, self.channel_name)
+
+    async def disconnect(self, code):
+        # TODO: log ip of disconnected client
+        logger.info("Client disconnected")
+
+    async def internal_message(self, event):
+        await self.send(event['text'])
 
 
-@channel_session_user
-def ws_receive(message):
-    from awx.main.access import consumer_access
-    user = message.user
-    raw_data = message.content['text']
-    data = json.loads(raw_data)
+class EventConsumer(AsyncJsonWebsocketConsumer):
+    async def connect(self):
+        user = self.scope['user']
+        if user and not user.is_anonymous:
+            await self.accept()
+            await self.send_json({"accept": True, "user": user.id})
+            # store the valid CSRF token from the cookie so we can compare it later
+            # on ws_receive
+            cookie_token = self.scope['cookies'].get('csrftoken')
+            if cookie_token:
+                self.scope['session'][XRF_KEY] = cookie_token
+        else:
+            logger.error("Request user is not authenticated to use websocket.")
+            # TODO: Carry over from channels 1 implementation
+            # We should never .accept() the client and close without sending a close message
+            await self.accept()
+            await self.send_json({"close": True})
+            await self.close()
 
-    xrftoken = data.get('xrftoken')
-    if (
-        not xrftoken or
-        XRF_KEY not in message.channel_session or
-        xrftoken != message.channel_session[XRF_KEY]
-    ):
-        logger.error(
-            "access denied to channel, XRF mismatch for {}".format(user.username)
-        )
-        message.reply_channel.send({
-            "text": json.dumps({"error": "access denied to channel"})
-        })
+    @database_sync_to_async
+    def user_can_see_object_id(self, user_access, oid):
+        # At this point user is a channels.auth.UserLazyObject object
+        # This causes problems with our generic role permissions checking.
+        # Specifically, type(user) != User
+        # Therefore, get the "real" User objects from the database before
+        # calling the access permission methods
+        user_access.user = User.objects.get(id=user_access.user.id)
+        res = user_access.get_queryset().filter(pk=oid).exists()
+        return res
+
+    async def receive_json(self, data):
+        from awx.main.access import consumer_access
+        user = self.scope['user']
+        xrftoken = data.get('xrftoken')
+        if (
+            not xrftoken or
+            XRF_KEY not in self.scope["session"] or
+            xrftoken != self.scope["session"][XRF_KEY]
+        ):
+            logger.error(f"access denied to channel, XRF mismatch for {user.username}")
+            await self.send_json({"error": "access denied to channel"})
+            return
+
+        if 'groups' in data:
+            groups = data['groups']
+            new_groups = set()
+            current_groups = set(self.scope['session'].pop('groups') if 'groups' in self.scope['session'] else [])
+            for group_name,v in groups.items():
+                if type(v) is list:
+                    for oid in v:
+                        name = '{}-{}'.format(group_name, oid)
+                        access_cls = consumer_access(group_name)
+                        if access_cls is not None:
+                            user_access = access_cls(user)
+                            if not await self.user_can_see_object_id(user_access, oid):
+                                await self.send_json({"error": "access denied to channel {0} for resource id {1}".format(group_name, oid)})
+                                continue
+                        new_groups.add(name)
+                else:
+                    if group_name == settings.BROADCAST_WEBSOCKET_GROUP_NAME:
+                        logger.warn("Non-priveleged client asked to join broadcast group!")
+                        return
+
+                    new_groups.add(group_name)
+
+            old_groups = current_groups - new_groups
+            for group_name in old_groups:
+                await self.channel_layer.group_discard(
+                    group_name,
+                    self.channel_name,
+                )
+
+            new_groups_exclusive = new_groups - current_groups
+            for group_name in new_groups_exclusive:
+                await self.channel_layer.group_add(
+                    group_name,
+                    self.channel_name
+                )
+            logger.debug(f"Channel {self.channel_name} left groups {old_groups} and joined {new_groups_exclusive}")
+            self.scope['session']['groups'] = new_groups
+            await self.send_json({
+                "groups_current": list(new_groups),
+                "groups_left": list(old_groups),
+                "groups_joined": list(new_groups_exclusive)
+            })
+
+    async def internal_message(self, event):
+        await self.send(event['text'])
+
+
+def run_sync(func):
+    event_loop = asyncio.new_event_loop()
+    event_loop.run_until_complete(func)
+    event_loop.close()
+
+
+def _dump_payload(payload):
+    try:
+        return json.dumps(payload, cls=DjangoJSONEncoder)
+    except ValueError:
+        logger.error("Invalid payload to emit")
+        return None
+
+
+async def emit_channel_notification_async(group, payload):
+    from awx.main.wsbroadcast import wrap_broadcast_msg # noqa
+
+    payload_dumped = _dump_payload(payload)
+    if payload_dumped is None:
         return
 
-    if 'groups' in data:
-        discard_groups(message)
-        groups = data['groups']
-        current_groups = set(message.channel_session.pop('groups') if 'groups' in message.channel_session else [])
-        for group_name,v in groups.items():
-            if type(v) is list:
-                for oid in v:
-                    name = '{}-{}'.format(group_name, oid)
-                    access_cls = consumer_access(group_name)
-                    if access_cls is not None:
-                        user_access = access_cls(user)
-                        if not user_access.get_queryset().filter(pk=oid).exists():
-                            message.reply_channel.send({"text": json.dumps(
-                                {"error": "access denied to channel {0} for resource id {1}".format(group_name, oid)})})
-                            continue
-                    current_groups.add(name)
-                    Group(name).add(message.reply_channel)
-            else:
-                current_groups.add(group_name)
-                Group(group_name).add(message.reply_channel)
-        message.channel_session['groups'] = list(current_groups)
+    channel_layer = get_channel_layer()
+    await channel_layer.group_send(
+        group,
+        {
+            "type": "internal.message",
+            "text": payload_dumped
+        },
+    )
+
+    await channel_layer.group_send(
+        settings.BROADCAST_WEBSOCKET_GROUP_NAME,
+        {
+            "type": "internal.message",
+            "text": wrap_broadcast_msg(group, payload_dumped),
+        },
+    )
 
 
 def emit_channel_notification(group, payload):
-    try:
-        Group(group).send({"text": json.dumps(payload, cls=DjangoJSONEncoder)})
-    except ValueError:
-        logger.error("Invalid payload emitting channel {} on topic: {}".format(group, payload))
+    from awx.main.wsbroadcast import wrap_broadcast_msg # noqa
+
+    payload_dumped = _dump_payload(payload)
+    if payload_dumped is None:
+        return
+
+    channel_layer = get_channel_layer()
+
+    run_sync(channel_layer.group_send(
+        group,
+        {
+            "type": "internal.message",
+            "text": payload_dumped
+        },
+    ))
+
+    run_sync(channel_layer.group_send(
+        settings.BROADCAST_WEBSOCKET_GROUP_NAME,
+        {
+            "type": "internal.message",
+            "text": wrap_broadcast_msg(group, payload_dumped),
+        },
+    ))

awx/main/db/profiled_pg/base.py

@@ -64,7 +64,7 @@ class RecordedQueryLog(object):
             if not os.path.isdir(self.dest):
                 os.makedirs(self.dest)
             progname = ' '.join(sys.argv)
-            for match in ('uwsgi', 'dispatcher', 'callback_receiver', 'runworker'):
+            for match in ('uwsgi', 'dispatcher', 'callback_receiver', 'wsbroadcast'):
                 if match in progname:
                     progname = match
                     break

awx/main/dispatch/__init__.py

@@ -1,5 +1,85 @@
+import psycopg2
+import select
+import sys
+import logging
+
+from contextlib import contextmanager
+
 from django.conf import settings
 
 
+NOT_READY = ([], [], [])
+if 'run_callback_receiver' in sys.argv:
+    logger = logging.getLogger('awx.main.commands.run_callback_receiver')
+else:
+    logger = logging.getLogger('awx.main.dispatch')
+
+
 def get_local_queuename():
     return settings.CLUSTER_HOST_ID
+
+
+class PubSub(object):
+    def __init__(self, conn):
+        assert conn.autocommit, "Connection must be in autocommit mode."
+        self.conn = conn
+
+    def listen(self, channel):
+        with self.conn.cursor() as cur:
+            cur.execute('LISTEN "%s";' % channel)
+
+    def unlisten(self, channel):
+        with self.conn.cursor() as cur:
+            cur.execute('UNLISTEN "%s";' % channel)
+
+    def notify(self, channel, payload):
+        with self.conn.cursor() as cur:
+            cur.execute('SELECT pg_notify(%s, %s);', (channel, payload))
+
+    def get_event(self, select_timeout=0):
+        # poll the connection, then return one event, if we have one.  Else
+        # return None.
+        select.select([self.conn], [], [], select_timeout)
+        self.conn.poll()
+        if self.conn.notifies:
+            return self.conn.notifies.pop(0)
+
+    def get_events(self, select_timeout=0):
+        # Poll the connection and return all events, if there are any.  Else
+        # return None.
+        select.select([self.conn], [], [], select_timeout) # redundant?
+        self.conn.poll()
+        events = []
+        while self.conn.notifies:
+            events.append(self.conn.notifies.pop(0))
+        if events:
+            return events
+
+    def events(self, select_timeout=5, yield_timeouts=False):
+        while True:
+            if select.select([self.conn], [], [], select_timeout) == NOT_READY:
+                if yield_timeouts:
+                    yield None
+            else:
+                self.conn.poll()
+                while self.conn.notifies:
+                    yield self.conn.notifies.pop(0)
+
+    def close(self):
+        self.conn.close()
+
+
+@contextmanager
+def pg_bus_conn():
+    conf = settings.DATABASES['default']
+    conn = psycopg2.connect(dbname=conf['NAME'],
+                            host=conf['HOST'],
+                            user=conf['USER'],
+                            password=conf['PASSWORD'])
+    # Django connection.cursor().connection doesn't have autocommit=True on
+    conn.set_session(autocommit=True)
+    pubsub = PubSub(conn)
+    yield pubsub
+    conn.close()
+
+

awx/main/dispatch/control.py

@@ -1,11 +1,11 @@
 import logging
-import socket
-
-from django.conf import settings
+import string
+import random
+import json
 
 from awx.main.dispatch import get_local_queuename
-from awx.main.dispatch.kombu import Connection
-from kombu import Queue, Exchange, Producer, Consumer
+
+from . import pg_bus_conn
 
 logger = logging.getLogger('awx.main.dispatch')
 
@@ -20,15 +20,10 @@ class Control(object):
             raise RuntimeError('{} must be in {}'.format(service, self.services))
         self.service = service
         self.queuename = host or get_local_queuename()
-        self.queue = Queue(self.queuename, Exchange(self.queuename), routing_key=self.queuename)
 
     def publish(self, msg, conn, **kwargs):
-        producer = Producer(
-            exchange=self.queue.exchange,
-            channel=conn,
-            routing_key=self.queuename
-        )
-        producer.publish(msg, expiration=5, **kwargs)
+        # TODO: delete this method??
+        raise RuntimeError("Publish called?!")
 
     def status(self, *args, **kwargs):
         return self.control_with_reply('status', *args, **kwargs)
@@ -36,24 +31,29 @@ class Control(object):
     def running(self, *args, **kwargs):
         return self.control_with_reply('running', *args, **kwargs)
 
+    @classmethod
+    def generate_reply_queue_name(cls):
+        letters = string.ascii_lowercase
+        return 'reply_to_{}'.format(''.join(random.choice(letters) for i in range(8)))
+
     def control_with_reply(self, command, timeout=5):
         logger.warn('checking {} {} for {}'.format(self.service, command, self.queuename))
-        reply_queue = Queue(name="amq.rabbitmq.reply-to")
+        reply_queue = Control.generate_reply_queue_name()
         self.result = None
-        with Connection(settings.BROKER_URL) as conn:
-            with Consumer(conn, reply_queue, callbacks=[self.process_message], no_ack=True):
-                self.publish({'control': command}, conn, reply_to='amq.rabbitmq.reply-to')
-                try:
-                    conn.drain_events(timeout=timeout)
-                except socket.timeout:
-                    logger.error('{} did not reply within {}s'.format(self.service, timeout))
-                    raise
-        return self.result
+
+        with pg_bus_conn() as conn:
+            conn.listen(reply_queue)
+            conn.notify(self.queuename,
+                        json.dumps({'control': command, 'reply_to': reply_queue}))
+
+            for reply in conn.events(select_timeout=timeout, yield_timeouts=True):
+                if reply is None:
+                    logger.error(f'{self.service} did not reply within {timeout}s')
+                    raise RuntimeError("{self.service} did not reply within {timeout}s")
+                break
+
+        return json.loads(reply.payload)
 
     def control(self, msg, **kwargs):
-        with Connection(settings.BROKER_URL) as conn:
-            self.publish(msg, conn)
-
-    def process_message(self, body, message):
-        self.result = body
-        message.ack()
+        with pg_bus_conn() as conn:
+            conn.notify(self.queuename, json.dumps(msg))

awx/main/dispatch/kombu.py

@@ -1,42 +0,0 @@
-from amqp.exceptions import PreconditionFailed
-from django.conf import settings
-from kombu.connection import Connection as KombuConnection
-from kombu.transport import pyamqp
-
-import logging
-
-logger = logging.getLogger('awx.main.dispatch')
-
-
-__all__ = ['Connection']
-
-
-class Connection(KombuConnection):
-
-    def __init__(self, *args, **kwargs):
-        super(Connection, self).__init__(*args, **kwargs)
-        class _Channel(pyamqp.Channel):
-
-            def queue_declare(self, queue, *args, **kwargs):
-                kwargs['durable'] = settings.BROKER_DURABILITY
-                try:
-                    return super(_Channel, self).queue_declare(queue, *args, **kwargs)
-                except PreconditionFailed as e:
-                    if "inequivalent arg 'durable'" in getattr(e, 'reply_text', None):
-                        logger.error(
-                            'queue {} durability is not {}, deleting and recreating'.format(
-
-                                queue,
-                                kwargs['durable']
-                            )
-                        )
-                        self.queue_delete(queue)
-                    return super(_Channel, self).queue_declare(queue, *args, **kwargs)
-
-        class _Connection(pyamqp.Connection):
-            Channel = _Channel
-
-        class _Transport(pyamqp.Transport):
-            Connection = _Connection
-
-        self.transport_cls = _Transport

awx/main/dispatch/publish.py

@@ -1,12 +1,12 @@
 import inspect
 import logging
 import sys
+import json
 from uuid import uuid4
 
 from django.conf import settings
-from kombu import Exchange, Producer
 
-from awx.main.dispatch.kombu import Connection
+from . import pg_bus_conn
 
 logger = logging.getLogger('awx.main.dispatch')
 
@@ -39,24 +39,22 @@ class task:
     add.apply_async([1, 1])
     Adder.apply_async([1, 1])
 
-    # Tasks can also define a specific target queue or exchange type:
+    # Tasks can also define a specific target queue or use the special fan-out queue tower_broadcast:
 
     @task(queue='slow-tasks')
     def snooze():
         time.sleep(10)
 
-    @task(queue='tower_broadcast', exchange_type='fanout')
+    @task(queue='tower_broadcast')
     def announce():
         print("Run this everywhere!")
     """
 
-    def __init__(self, queue=None, exchange_type=None):
+    def __init__(self, queue=None):
         self.queue = queue
-        self.exchange_type = exchange_type
 
     def __call__(self, fn=None):
         queue = self.queue
-        exchange_type = self.exchange_type
 
         class PublisherMixin(object):
 
@@ -73,9 +71,12 @@ class task:
                 kwargs = kwargs or {}
                 queue = (
                     queue or
-                    getattr(cls.queue, 'im_func', cls.queue) or
-                    settings.CELERY_DEFAULT_QUEUE
+                    getattr(cls.queue, 'im_func', cls.queue)
                 )
+                if not queue:
+                    msg = f'{cls.name}: Queue value required and may not me None'
+                    logger.error(msg)
+                    raise ValueError(msg)
                 obj = {
                     'uuid': task_id,
                     'args': args,
@@ -86,21 +87,8 @@ class task:
                 if callable(queue):
                     queue = queue()
                 if not settings.IS_TESTING(sys.argv):
-                    with Connection(settings.BROKER_URL) as conn:
-                        exchange = Exchange(queue, type=exchange_type or 'direct')
-                        producer = Producer(conn)
-                        logger.debug('publish {}({}, queue={})'.format(
-                            cls.name,
-                            task_id,
-                            queue
-                        ))
-                        producer.publish(obj,
-                                         serializer='json',
-                                         compression='bzip2',
-                                         exchange=exchange,
-                                         declare=[exchange],
-                                         delivery_mode="persistent",
-                                         routing_key=queue)
+                    with pg_bus_conn() as conn:
+                        conn.notify(queue, json.dumps(obj))
                 return (obj, queue)
 
         # If the object we're wrapping *is* a class (e.g., RunJob), return

awx/main/dispatch/worker/__init__.py

@@ -1,3 +1,3 @@
-from .base import AWXConsumer, BaseWorker  # noqa
+from .base import AWXConsumerRedis, AWXConsumerPG, BaseWorker  # noqa
 from .callback import CallbackBrokerWorker  # noqa
 from .task import TaskWorker  # noqa

awx/main/dispatch/worker/base.py

@@ -5,14 +5,17 @@ import os
 import logging
 import signal
 import sys
+import redis
+import json
+import psycopg2
 from uuid import UUID
 from queue import Empty as QueueEmpty
 
 from django import db
-from kombu import Producer
-from kombu.mixins import ConsumerMixin
+from django.conf import settings
 
 from awx.main.dispatch.pool import WorkerPool
+from awx.main.dispatch import pg_bus_conn
 
 if 'run_callback_receiver' in sys.argv:
     logger = logging.getLogger('awx.main.commands.run_callback_receiver')
@@ -37,10 +40,11 @@ class WorkerSignalHandler:
         self.kill_now = True
 
 
-class AWXConsumer(ConsumerMixin):
+class AWXConsumerBase(object):
+    def __init__(self, name, worker, queues=[], pool=None):
+        self.should_stop = False
 
-    def __init__(self, name, connection, worker, queues=[], pool=None):
-        self.connection = connection
+        self.name = name
         self.total_messages = 0
         self.queues = queues
         self.worker = worker
@@ -49,25 +53,15 @@ class AWXConsumer(ConsumerMixin):
             self.pool = WorkerPool()
         self.pool.init_workers(self.worker.work_loop)
 
-    def get_consumers(self, Consumer, channel):
-        logger.debug(self.listening_on)
-        return [Consumer(queues=self.queues, accept=['json'],
-                         callbacks=[self.process_task])]
-
     @property
     def listening_on(self):
-        return 'listening on {}'.format([
-            '{} [{}]'.format(q.name, q.exchange.type) for q in self.queues
-        ])
+        return f'listening on {self.queues}'
 
-    def control(self, body, message):
-        logger.warn('Consumer received control message {}'.format(body))
+    def control(self, body):
+        logger.warn(body)
         control = body.get('control')
         if control in ('status', 'running'):
-            producer = Producer(
-                channel=self.connection,
-                routing_key=message.properties['reply_to']
-            )
+            reply_queue = body['reply_to']
             if control == 'status':
                 msg = '\n'.join([self.listening_on, self.pool.debug()])
             elif control == 'running':
@@ -75,20 +69,21 @@ class AWXConsumer(ConsumerMixin):
                 for worker in self.pool.workers:
                     worker.calculate_managed_tasks()
                     msg.extend(worker.managed_tasks.keys())
-            producer.publish(msg)
+
+            with pg_bus_conn() as conn:
+                conn.notify(reply_queue, json.dumps(msg))
         elif control == 'reload':
             for worker in self.pool.workers:
                 worker.quit()
         else:
             logger.error('unrecognized control message: {}'.format(control))
-        message.ack()
 
-    def process_task(self, body, message):
+    def process_task(self, body):
         if 'control' in body:
             try:
-                return self.control(body, message)
+                return self.control(body)
             except Exception:
-                logger.exception("Exception handling control message:")
+                logger.exception(f"Exception handling control message: {body}")
                 return
         if len(self.pool):
             if "uuid" in body and body['uuid']:
@@ -102,21 +97,54 @@ class AWXConsumer(ConsumerMixin):
             queue = 0
         self.pool.write(queue, body)
         self.total_messages += 1
-        message.ack()
 
     def run(self, *args, **kwargs):
         signal.signal(signal.SIGINT, self.stop)
         signal.signal(signal.SIGTERM, self.stop)
         self.worker.on_start()
-        super(AWXConsumer, self).run(*args, **kwargs)
+
+        # Child should implement other things here
 
     def stop(self, signum, frame):
-        self.should_stop = True  # this makes the kombu mixin stop consuming
+        self.should_stop = True
         logger.warn('received {}, stopping'.format(signame(signum)))
         self.worker.on_stop()
         raise SystemExit()
 
 
+class AWXConsumerRedis(AWXConsumerBase):
+    def run(self, *args, **kwargs):
+        super(AWXConsumerRedis, self).run(*args, **kwargs)
+
+        queue = redis.Redis.from_url(settings.BROKER_URL)
+        while True:
+            res = queue.blpop(self.queues)
+            res = json.loads(res[1])
+            self.process_task(res)
+            if self.should_stop:
+                return
+
+
+class AWXConsumerPG(AWXConsumerBase):
+    def run(self, *args, **kwargs):
+        super(AWXConsumerPG, self).run(*args, **kwargs)
+
+        logger.warn(f"Running worker {self.name} listening to queues {self.queues}")
+
+        while True:
+            try:
+                with pg_bus_conn() as conn:
+                    for queue in self.queues:
+                        conn.listen(queue)
+                    for e in conn.events():
+                        self.process_task(json.loads(e.payload))
+                    if self.should_stop:
+                        return
+            except psycopg2.InterfaceError:
+                logger.warn("Stale Postgres message bus connection, reconnecting")
+                continue
+
+
 class BaseWorker(object):
 
     def read(self, queue):

awx/main/management/commands/run_callback_receiver.py

@@ -3,10 +3,8 @@
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
-from kombu import Exchange, Queue
 
-from awx.main.dispatch.kombu import Connection
-from awx.main.dispatch.worker import AWXConsumer, CallbackBrokerWorker
+from awx.main.dispatch.worker import AWXConsumerRedis, CallbackBrokerWorker
 
 
 class Command(BaseCommand):
@@ -18,23 +16,15 @@ class Command(BaseCommand):
     help = 'Launch the job callback receiver'
 
     def handle(self, *arg, **options):
-        with Connection(settings.BROKER_URL) as conn:
-            consumer = None
-            try:
-                consumer = AWXConsumer(
-                    'callback_receiver',
-                    conn,
-                    CallbackBrokerWorker(),
-                    [
-                        Queue(
-                            settings.CALLBACK_QUEUE,
-                            Exchange(settings.CALLBACK_QUEUE, type='direct'),
-                            routing_key=settings.CALLBACK_QUEUE
-                        )
-                    ]
-                )
-                consumer.run()
-            except KeyboardInterrupt:
-                print('Terminating Callback Receiver')
-                if consumer:
-                    consumer.stop()
+        consumer = None
+        try:
+            consumer = AWXConsumerRedis(
+                'callback_receiver',
+                CallbackBrokerWorker(),
+                queues=[getattr(settings, 'CALLBACK_QUEUE', '')],
+            )
+            consumer.run()
+        except KeyboardInterrupt:
+            print('Terminating Callback Receiver')
+            if consumer:
+                consumer.stop()

awx/main/management/commands/run_dispatcher.py

@@ -6,14 +6,12 @@ from django.conf import settings
 from django.core.cache import cache as django_cache
 from django.core.management.base import BaseCommand
 from django.db import connection as django_connection
-from kombu import Exchange, Queue
 
 from awx.main.utils.handlers import AWXProxyHandler
 from awx.main.dispatch import get_local_queuename, reaper
 from awx.main.dispatch.control import Control
-from awx.main.dispatch.kombu import Connection
 from awx.main.dispatch.pool import AutoscalePool
-from awx.main.dispatch.worker import AWXConsumer, TaskWorker
+from awx.main.dispatch.worker import AWXConsumerPG, TaskWorker
 from awx.main.dispatch import periodic
 
 logger = logging.getLogger('awx.main.dispatch')
@@ -63,30 +61,16 @@ class Command(BaseCommand):
         # in cpython itself:
         # https://bugs.python.org/issue37429
         AWXProxyHandler.disable()
-        with Connection(settings.BROKER_URL) as conn:
-            try:
-                bcast = 'tower_broadcast_all'
-                queues = [
-                    Queue(q, Exchange(q), routing_key=q)
-                    for q in (settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()])
-                ]
-                queues.append(
-                    Queue(
-                        construct_bcast_queue_name(bcast),
-                        exchange=Exchange(bcast, type='fanout'),
-                        routing_key=bcast,
-                        reply=True
-                    )
-                )
-                consumer = AWXConsumer(
-                    'dispatcher',
-                    conn,
-                    TaskWorker(),
-                    queues,
-                    AutoscalePool(min_workers=4)
-                )
-                consumer.run()
-            except KeyboardInterrupt:
-                logger.debug('Terminating Task Dispatcher')
-                if consumer:
-                    consumer.stop()
+        try:
+            queues = ['tower_broadcast_all', get_local_queuename()]
+            consumer = AWXConsumerPG(
+                'dispatcher',
+                TaskWorker(),
+                queues,
+                AutoscalePool(min_workers=4)
+            )
+            consumer.run()
+        except KeyboardInterrupt:
+            logger.debug('Terminating Task Dispatcher')
+            if consumer:
+                consumer.stop()

awx/main/management/commands/run_wsbroadcast.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved.
+import logging
+import asyncio
+
+from django.core.management.base import BaseCommand
+
+from awx.main.wsbroadcast import BroadcastWebsocketManager
+
+
+logger = logging.getLogger('awx.main.wsbroadcast')
+
+
+class Command(BaseCommand):
+    help = 'Launch the websocket broadcaster'
+
+    def handle(self, *arg, **options):
+        try:
+            broadcast_websocket_mgr = BroadcastWebsocketManager()
+            task = broadcast_websocket_mgr.start()
+
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(task)
+        except KeyboardInterrupt:
+            logger.debug('Terminating Websocket Broadcaster')

awx/main/managers.py

@@ -3,6 +3,7 @@
 
 import sys
 import logging
+import os
 
 from django.db import models
 from django.conf import settings
@@ -114,7 +115,7 @@ class InstanceManager(models.Manager):
             return node[0]
         raise RuntimeError("No instance found with the current cluster host id")
 
-    def register(self, uuid=None, hostname=None):
+    def register(self, uuid=None, hostname=None, ip_address=None):
         if not uuid:
             uuid = settings.SYSTEM_UUID
         if not hostname:
@@ -122,13 +123,23 @@ class InstanceManager(models.Manager):
         with advisory_lock('instance_registration_%s' % hostname):
             instance = self.filter(hostname=hostname)
             if instance.exists():
-                return (False, instance[0])
-            instance = self.create(uuid=uuid, hostname=hostname, capacity=0)
+                instance = instance.get()
+                if instance.ip_address != ip_address:
+                    instance.ip_address = ip_address
+                    instance.save()
+                    return (True, instance)
+                else:
+                    return (False, instance)
+            instance = self.create(uuid=uuid,
+                                   hostname=hostname,
+                                   ip_address=ip_address,
+                                   capacity=0)
         return (True, instance)
 
     def get_or_register(self):
         if settings.AWX_AUTO_DEPROVISION_INSTANCES:
-            return self.register()
+            pod_ip = os.environ.get('MY_POD_IP')
+            return self.register(ip_address=pod_ip)
         else:
             return (False, self.me())
 

awx/main/migrations/0110_v370_instance_ip_address.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.8 on 2020-02-12 17:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0109_v370_job_template_organization_field'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instance',
+            name='ip_address',
+            field=models.CharField(blank=True, default=None, max_length=50, null=True, unique=True),
+        ),
+    ]

awx/main/migrations/0111_v370_delete_channelgroup.py

@@ -0,0 +1,16 @@
+# Generated by Django 2.2.8 on 2020-02-17 14:50
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0110_v370_instance_ip_address'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='ChannelGroup',
+        ),
+    ]

awx/main/models/__init__.py

@@ -58,7 +58,6 @@ from awx.main.models.workflow import (  # noqa
     WorkflowJob, WorkflowJobNode, WorkflowJobOptions, WorkflowJobTemplate,
     WorkflowJobTemplateNode, WorkflowApproval, WorkflowApprovalTemplate,
 )
-from awx.main.models.channels import ChannelGroup # noqa
 from awx.api.versioning import reverse
 from awx.main.models.oauth import ( # noqa
     OAuth2AccessToken, OAuth2Application

awx/main/models/channels.py

@@ -1,6 +0,0 @@
-from django.db import models
-
-
-class ChannelGroup(models.Model):
-    group = models.CharField(max_length=200, unique=True)
-    channels = models.TextField()

awx/main/models/ha.py

@@ -53,6 +53,13 @@ class Instance(HasPolicyEditsMixin, BaseModel):
 
     uuid = models.CharField(max_length=40)
     hostname = models.CharField(max_length=250, unique=True)
+    ip_address = models.CharField(
+        blank=True,
+        null=True,
+        default=None,
+        max_length=50,
+        unique=True,
+    )
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
     last_isolated_check = models.DateTimeField(

awx/main/models/unified_jobs.py

@@ -36,6 +36,7 @@ from awx.main.models.base import (
     NotificationFieldsModel,
     prevent_search
 )
+from awx.main.dispatch import get_local_queuename
 from awx.main.dispatch.control import Control as ControlDispatcher
 from awx.main.registrar import activity_stream_registrar
 from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin
@@ -1360,7 +1361,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
             timeout = 5
             try:
                 running = self.celery_task_id in ControlDispatcher(
-                    'dispatcher', self.execution_node
+                    'dispatcher', self.controller_node or self.execution_node
                 ).running(timeout=timeout)
             except socket.timeout:
                 logger.error('could not reach dispatcher on {} within {}s'.format(
@@ -1466,7 +1467,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         return r
 
     def get_queue_name(self):
-        return self.controller_node or self.execution_node or settings.CELERY_DEFAULT_QUEUE
+        return self.controller_node or self.execution_node or get_local_queuename()
 
     def is_isolated(self):
         return bool(self.controller_node)

awx/main/queue.py

@@ -4,15 +4,11 @@
 # Python
 import json
 import logging
-import os
+import redis
 
 # Django
 from django.conf import settings
 
-# Kombu
-from awx.main.dispatch.kombu import Connection
-from kombu import Exchange, Producer
-from kombu.serialization import registry
 
 __all__ = ['CallbackQueueDispatcher']
 
@@ -28,47 +24,12 @@ class AnsibleJSONEncoder(json.JSONEncoder):
         return super(AnsibleJSONEncoder, self).default(o)
 
 
-registry.register(
-    'json-ansible',
-    lambda obj: json.dumps(obj, cls=AnsibleJSONEncoder),
-    lambda obj: json.loads(obj),
-    content_type='application/json',
-    content_encoding='utf-8'
-)
-
-
 class CallbackQueueDispatcher(object):
 
     def __init__(self):
-        self.callback_connection = getattr(settings, 'BROKER_URL', None)
-        self.connection_queue = getattr(settings, 'CALLBACK_QUEUE', '')
-        self.connection = None
-        self.exchange = None
+        self.queue = getattr(settings, 'CALLBACK_QUEUE', '')
         self.logger = logging.getLogger('awx.main.queue.CallbackQueueDispatcher')
+        self.connection = redis.Redis.from_url(settings.BROKER_URL)
 
     def dispatch(self, obj):
-        if not self.callback_connection or not self.connection_queue:
-            return
-        active_pid = os.getpid()
-        for retry_count in range(4):
-            try:
-                if not hasattr(self, 'connection_pid'):
-                    self.connection_pid = active_pid
-                if self.connection_pid != active_pid:
-                    self.connection = None
-                if self.connection is None:
-                    self.connection = Connection(self.callback_connection)
-                    self.exchange = Exchange(self.connection_queue, type='direct')
-
-                producer = Producer(self.connection)
-                producer.publish(obj,
-                                 serializer='json-ansible',
-                                 compression='bzip2',
-                                 exchange=self.exchange,
-                                 declare=[self.exchange],
-                                 delivery_mode="persistent" if settings.PERSISTENT_CALLBACK_MESSAGES else "transient",
-                                 routing_key=self.connection_queue)
-                return
-            except Exception as e:
-                self.logger.info('Publish Job Event Exception: %r, retry=%d', e,
-                                 retry_count, exc_info=True)
+        self.connection.rpush(self.queue, json.dumps(obj, cls=AnsibleJSONEncoder))

awx/main/routing.py

@@ -1,8 +1,15 @@
-from channels.routing import route
+from django.conf.urls import url
+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+from . import consumers
 
-
-channel_routing = [
-    route("websocket.connect", "awx.main.consumers.ws_connect", path=r'^/websocket/$'),
-    route("websocket.disconnect", "awx.main.consumers.ws_disconnect", path=r'^/websocket/$'),
-    route("websocket.receive", "awx.main.consumers.ws_receive", path=r'^/websocket/$'),
+websocket_urlpatterns = [
+    url(r'websocket/$', consumers.EventConsumer),
+    url(r'websocket/broadcast/$', consumers.BroadcastConsumer),
 ]
+
+application = ProtocolTypeRouter({
+    'websocket': AuthMiddlewareStack(
+        URLRouter(websocket_urlpatterns)
+    ),
+})

awx/main/scheduler/tasks.py

@@ -5,11 +5,12 @@ import logging
 # AWX
 from awx.main.scheduler import TaskManager
 from awx.main.dispatch.publish import task
+from awx.main.dispatch import get_local_queuename
 
 logger = logging.getLogger('awx.main.scheduler')
 
 
-@task()
+@task(queue=get_local_queuename)
 def run_task_manager():
     logger.debug("Running Tower task manager.")
     TaskManager().schedule()

awx/main/signals.py

@@ -6,7 +6,6 @@ import contextlib
 import logging
 import threading
 import json
-import pkg_resources
 import sys
 
 # Django
@@ -593,16 +592,6 @@ def deny_orphaned_approvals(sender, instance, **kwargs):
 @receiver(post_save, sender=Session)
 def save_user_session_membership(sender, **kwargs):
     session = kwargs.get('instance', None)
-    if pkg_resources.get_distribution('channels').version >= '2':
-        # If you get into this code block, it means we upgraded channels, but
-        # didn't make the settings.SESSIONS_PER_USER feature work
-        raise RuntimeError(
-            'save_user_session_membership must be updated for channels>=2: '
-            'http://channels.readthedocs.io/en/latest/one-to-two.html#requirements'
-        )
-    if 'runworker' in sys.argv:
-        # don't track user session membership for websocket per-channel sessions
-        return
     if not session:
         return
     user_id = session.get_decoded().get(SESSION_KEY, None)

awx/main/tasks.py

@@ -151,7 +151,7 @@ def inform_cluster_of_shutdown():
         logger.exception('Encountered problem with normal shutdown signal.')
 
 
-@task()
+@task(queue=get_local_queuename)
 def apply_cluster_membership_policies():
     started_waiting = time.time()
     with advisory_lock('cluster_policy_lock', wait=True):
@@ -264,7 +264,7 @@ def apply_cluster_membership_policies():
         logger.debug('Cluster policy computation finished in {} seconds'.format(time.time() - started_compute))
 
 
-@task(queue='tower_broadcast_all', exchange_type='fanout')
+@task(queue='tower_broadcast_all')
 def handle_setting_changes(setting_keys):
     orig_len = len(setting_keys)
     for i in range(orig_len):
@@ -275,7 +275,7 @@ def handle_setting_changes(setting_keys):
     cache.delete_many(cache_keys)
 
 
-@task(queue='tower_broadcast_all', exchange_type='fanout')
+@task(queue='tower_broadcast_all')
 def delete_project_files(project_path):
     # TODO: possibly implement some retry logic
     lock_file = project_path + '.lock'
@@ -293,7 +293,7 @@ def delete_project_files(project_path):
             logger.exception('Could not remove lock file {}'.format(lock_file))
 
 
-@task(queue='tower_broadcast_all', exchange_type='fanout')
+@task(queue='tower_broadcast_all')
 def profile_sql(threshold=1, minutes=1):
     if threshold == 0:
         cache.delete('awx-profile-sql-threshold')
@@ -307,7 +307,7 @@ def profile_sql(threshold=1, minutes=1):
         logger.error('SQL QUERIES >={}s ENABLED FOR {} MINUTE(S)'.format(threshold, minutes))
 
 
-@task()
+@task(queue=get_local_queuename)
 def send_notifications(notification_list, job_id=None):
     if not isinstance(notification_list, list):
         raise TypeError("notification_list should be of type list")
@@ -336,7 +336,7 @@ def send_notifications(notification_list, job_id=None):
                 logger.exception('Error saving notification {} result.'.format(notification.id))
 
 
-@task()
+@task(queue=get_local_queuename)
 def gather_analytics():
     from awx.conf.models import Setting
     from rest_framework.fields import DateTimeField
@@ -492,7 +492,7 @@ def awx_isolated_heartbeat():
         isolated_manager.IsolatedManager(CallbackQueueDispatcher.dispatch).health_check(isolated_instance_qs)
 
 
-@task()
+@task(queue=get_local_queuename)
 def awx_periodic_scheduler():
     with advisory_lock('awx_periodic_scheduler_lock', wait=False) as acquired:
         if acquired is False:
@@ -549,7 +549,7 @@ def awx_periodic_scheduler():
         state.save()
 
 
-@task()
+@task(queue=get_local_queuename)
 def handle_work_success(task_actual):
     try:
         instance = UnifiedJob.get_instance_by_type(task_actual['type'], task_actual['id'])
@@ -562,7 +562,7 @@ def handle_work_success(task_actual):
     schedule_task_manager()
 
 
-@task()
+@task(queue=get_local_queuename)
 def handle_work_error(task_id, *args, **kwargs):
     subtasks = kwargs.get('subtasks', None)
     logger.debug('Executing error task id %s, subtasks: %s' % (task_id, str(subtasks)))
@@ -602,7 +602,7 @@ def handle_work_error(task_id, *args, **kwargs):
         pass
 
 
-@task()
+@task(queue=get_local_queuename)
 def update_inventory_computed_fields(inventory_id):
     '''
     Signal handler and wrapper around inventory.update_computed_fields to
@@ -644,7 +644,7 @@ def update_smart_memberships_for_inventory(smart_inventory):
     return False
 
 
-@task()
+@task(queue=get_local_queuename)
 def update_host_smart_inventory_memberships():
     smart_inventories = Inventory.objects.filter(kind='smart', host_filter__isnull=False, pending_deletion=False)
     changed_inventories = set([])
@@ -660,7 +660,7 @@ def update_host_smart_inventory_memberships():
         smart_inventory.update_computed_fields()
 
 
-@task()
+@task(queue=get_local_queuename)
 def delete_inventory(inventory_id, user_id, retries=5):
     # Delete inventory as user
     if user_id is None:
@@ -1478,7 +1478,7 @@ class BaseTask(object):
 
 
 
-@task()
+@task(queue=get_local_queuename)
 class RunJob(BaseTask):
     '''
     Run a job using ansible-playbook.
@@ -1911,7 +1911,7 @@ class RunJob(BaseTask):
                 update_inventory_computed_fields.delay(inventory.id)
 
 
-@task()
+@task(queue=get_local_queuename)
 class RunProjectUpdate(BaseTask):
 
     model = ProjectUpdate
@@ -2321,7 +2321,7 @@ class RunProjectUpdate(BaseTask):
         return getattr(settings, 'AWX_PROOT_ENABLED', False)
 
 
-@task()
+@task(queue=get_local_queuename)
 class RunInventoryUpdate(BaseTask):
 
     model = InventoryUpdate
@@ -2589,7 +2589,7 @@ class RunInventoryUpdate(BaseTask):
             )
 
 
-@task()
+@task(queue=get_local_queuename)
 class RunAdHocCommand(BaseTask):
     '''
     Run an ad hoc command using ansible.
@@ -2779,7 +2779,7 @@ class RunAdHocCommand(BaseTask):
             isolated_manager_instance.cleanup()
 
 
-@task()
+@task(queue=get_local_queuename)
 class RunSystemJob(BaseTask):
 
     model = SystemJob
@@ -2853,11 +2853,16 @@ def _reconstruct_relationships(copy_mapping):
         new_obj.save()
 
 
-@task()
+@task(queue=get_local_queuename)
 def deep_copy_model_obj(
     model_module, model_name, obj_pk, new_obj_pk,
-    user_pk, sub_obj_list, permission_check_func=None
+    user_pk, uuid, permission_check_func=None
 ):
+    sub_obj_list = cache.get(uuid)
+    if sub_obj_list is None:
+        logger.error('Deep copy {} from {} to {} failed unexpectedly.'.format(model_name, obj_pk, new_obj_pk))
+        return
+
     logger.debug('Deep copy {} from {} to {}.'.format(model_name, obj_pk, new_obj_pk))
     from awx.api.generics import CopyAPIView
     from awx.main.signals import disable_activity_stream

awx/main/tests/functional/api/test_settings.py

@@ -8,7 +8,6 @@ import os
 import time
 
 from django.conf import settings
-from kombu.utils.url import parse_url
 
 # Mock
 from unittest import mock
@@ -386,15 +385,3 @@ def test_saml_x509cert_validation(patch, get, admin, headers):
         }
     })
     assert resp.status_code == 200
-
-
-@pytest.mark.django_db
-def test_broker_url_with_special_characters():
-    settings.BROKER_URL = 'amqp://guest:a@ns:ibl3#@rabbitmq:5672//'
-    url = parse_url(settings.BROKER_URL)
-    assert url['transport'] == 'amqp'
-    assert url['hostname'] == 'rabbitmq'
-    assert url['port'] == 5672
-    assert url['userid'] == 'guest'
-    assert url['password'] == 'a@ns:ibl3#'
-    assert url['virtual_host'] == '/'

awx/main/tests/functional/models/test_schedule.py

@@ -103,7 +103,8 @@ class TestComputedFields:
             Schedule.objects.filter(pk=s.pk).update(next_run=old_next_run)
             s.next_run = old_next_run
             prior_modified = s.modified
-            s.update_computed_fields()
+            with mock.patch('awx.main.models.schedules.emit_channel_notification'):
+                s.update_computed_fields()
             assert s.next_run != old_next_run
             assert s.modified == prior_modified
 
@@ -133,7 +134,8 @@ class TestComputedFields:
             assert s.next_run is None
             assert job_template.next_schedule is None
             s.rrule = self.distant_rrule
-            s.update_computed_fields()
+            with mock.patch('awx.main.models.schedules.emit_channel_notification'):
+                s.update_computed_fields()
             assert s.next_run is not None
             assert job_template.next_schedule == s
 

awx/main/tests/functional/test_dispatch.py

@@ -15,6 +15,15 @@ from awx.main.dispatch.publish import task
 from awx.main.dispatch.worker import BaseWorker, TaskWorker
 
 
+'''
+Prevent logger.<warn, debug, error> calls from triggering database operations
+'''
+@pytest.fixture(autouse=True)
+def _disable_database_settings(mocker):
+    m = mocker.patch('awx.conf.settings.SettingsWrapper.all_supported_settings', new_callable=mock.PropertyMock)
+    m.return_value = []
+
+
 def restricted(a, b):
     raise AssertionError("This code should not run because it isn't decorated with @task")
 
@@ -324,22 +333,23 @@ class TestTaskPublisher:
         assert Adder().run(2, 2) == 4
 
     def test_function_apply_async(self):
-        message, queue = add.apply_async([2, 2])
+        message, queue = add.apply_async([2, 2], queue='foobar')
         assert message['args'] == [2, 2]
         assert message['kwargs'] == {}
         assert message['task'] == 'awx.main.tests.functional.test_dispatch.add'
-        assert queue == 'awx_private_queue'
+        assert queue == 'foobar'
 
     def test_method_apply_async(self):
-        message, queue = Adder.apply_async([2, 2])
+        message, queue = Adder.apply_async([2, 2], queue='foobar')
         assert message['args'] == [2, 2]
         assert message['kwargs'] == {}
         assert message['task'] == 'awx.main.tests.functional.test_dispatch.Adder'
-        assert queue == 'awx_private_queue'
+        assert queue == 'foobar'
 
-    def test_apply_with_queue(self):
-        message, queue = add.apply_async([2, 2], queue='abc123')
-        assert queue == 'abc123'
+    def test_apply_async_queue_required(self):
+        with pytest.raises(ValueError) as e:
+            message, queue = add.apply_async([2, 2])
+        assert "awx.main.tests.functional.test_dispatch.add: Queue value required and may not me None" == e.value.args[0]
 
     def test_queue_defined_in_task_decorator(self):
         message, queue = multiply.apply_async([2, 2])

awx/main/wsbroadcast.py

@@ -0,0 +1,190 @@
+import json
+import logging
+import asyncio
+
+import aiohttp
+from aiohttp import client_exceptions
+
+from channels.layers import get_channel_layer
+
+from django.conf import settings
+from django.apps import apps
+from django.core.serializers.json import DjangoJSONEncoder
+
+from awx.main.analytics.broadcast_websocket import (
+    BroadcastWebsocketStats,
+    BroadcastWebsocketStatsManager,
+)
+
+
+logger = logging.getLogger('awx.main.wsbroadcast')
+
+
+def wrap_broadcast_msg(group, message: str):
+    # TODO: Maybe wrap as "group","message" so that we don't need to
+    # encode/decode as json.
+    return json.dumps(dict(group=group, message=message), cls=DjangoJSONEncoder)
+
+
+def unwrap_broadcast_msg(payload: dict):
+    return (payload['group'], payload['message'])
+
+
+def get_broadcast_hosts():
+    Instance = apps.get_model('main', 'Instance')
+    instances = Instance.objects.filter(rampart_groups__controller__isnull=True) \
+                                .exclude(hostname=Instance.objects.me().hostname) \
+                                .order_by('hostname') \
+                                .values('hostname', 'ip_address') \
+                                .distinct()
+    return [i['ip_address'] or i['hostname'] for i in instances]
+
+
+def get_local_host():
+    Instance = apps.get_model('main', 'Instance')
+    return Instance.objects.me().hostname
+
+
+class WebsocketTask():
+    def __init__(self,
+                 name,
+                 event_loop,
+                 stats: BroadcastWebsocketStats,
+                 remote_host: str,
+                 remote_port: int = settings.BROADCAST_WEBSOCKET_PORT,
+                 protocol: str = settings.BROADCAST_WEBSOCKET_PROTOCOL,
+                 verify_ssl: bool = settings.BROADCAST_WEBSOCKET_VERIFY_CERT,
+                 endpoint: str = 'broadcast'):
+        self.name = name
+        self.event_loop = event_loop
+        self.stats = stats
+        self.remote_host = remote_host
+        self.remote_port = remote_port
+        self.endpoint = endpoint
+        self.protocol = protocol
+        self.verify_ssl = verify_ssl
+        self.channel_layer = None
+
+    async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse):
+        raise RuntimeError("Implement me")
+
+    async def connect(self, attempt):
+        from awx.main.consumers import WebsocketSecretAuthHelper # noqa
+        logger.debug(f"{self.name} connect attempt {attempt} to {self.remote_host}")
+
+        '''
+        Can not put get_channel_layer() in the init code because it is in the init
+        path of channel layers i.e. RedisChannelLayer() calls our init code.
+        '''
+        if not self.channel_layer:
+            self.channel_layer = get_channel_layer()
+
+        try:
+            if attempt > 0:
+                await asyncio.sleep(settings.BROADCAST_WEBSOCKET_RECONNECT_RETRY_RATE_SECONDS)
+        except asyncio.CancelledError:
+            logger.warn(f"{self.name} connection to {self.remote_host} cancelled")
+            raise
+
+        uri = f"{self.protocol}://{self.remote_host}:{self.remote_port}/websocket/{self.endpoint}/"
+        timeout = aiohttp.ClientTimeout(total=10)
+
+        secret_val = WebsocketSecretAuthHelper.construct_secret()
+        try:
+            async with aiohttp.ClientSession(headers={'secret': secret_val},
+                                             timeout=timeout) as session:
+                async with session.ws_connect(uri, ssl=self.verify_ssl) as websocket:
+                    self.stats.record_connection_established()
+                    attempt = 0
+                    await self.run_loop(websocket)
+        except asyncio.CancelledError:
+            # TODO: Check if connected and disconnect
+            # Possibly use run_until_complete() if disconnect is async
+            logger.warn(f"{self.name} connection to {self.remote_host} cancelled")
+            self.stats.record_connection_lost()
+            raise
+        except client_exceptions.ClientConnectorError as e:
+            logger.warn(f"Failed to connect to {self.remote_host}: '{e}'. Reconnecting ...")
+            self.stats.record_connection_lost()
+            self.start(attempt=attempt + 1)
+        except asyncio.TimeoutError:
+            logger.warn(f"Timeout while trying to connect to {self.remote_host}. Reconnecting ...")
+            self.stats.record_connection_lost()
+            self.start(attempt=attempt + 1)
+        except Exception as e:
+            # Early on, this is our canary. I'm not sure what exceptions we can really encounter.
+            logger.warn(f"Websocket broadcast client exception {type(e)} {e}")
+            self.stats.record_connection_lost()
+            self.start(attempt=attempt + 1)
+
+    def start(self, attempt=0):
+        self.async_task = self.event_loop.create_task(self.connect(attempt=attempt))
+
+    def cancel(self):
+        self.async_task.cancel()
+
+
+class BroadcastWebsocketTask(WebsocketTask):
+    async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse):
+        async for msg in websocket:
+            self.stats.record_message_received()
+
+            if msg.type == aiohttp.WSMsgType.ERROR:
+                break
+            elif msg.type == aiohttp.WSMsgType.TEXT:
+                try:
+                    payload = json.loads(msg.data)
+                except json.JSONDecodeError:
+                    logmsg = "Failed to decode broadcast message"
+                    if logger.isEnabledFor(logging.DEBUG):
+                        logmsg = "{} {}".format(logmsg, payload)
+                    logger.warn(logmsg)
+                    continue
+
+                (group, message) = unwrap_broadcast_msg(payload)
+
+                await self.channel_layer.group_send(group, {"type": "internal.message", "text": message})
+
+
+class BroadcastWebsocketManager(object):
+    def __init__(self):
+        self.event_loop = asyncio.get_event_loop()
+        self.broadcast_tasks = dict()
+        # parallel dict to broadcast_tasks that tracks stats
+        self.local_hostname = get_local_host()
+        self.stats_mgr = BroadcastWebsocketStatsManager(self.event_loop, self.local_hostname)
+
+    async def run_per_host_websocket(self):
+
+        while True:
+            future_remote_hosts = get_broadcast_hosts()
+            current_remote_hosts = self.broadcast_tasks.keys()
+            deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts)
+            new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts)
+
+            if deleted_remote_hosts:
+                logger.warn(f"{self.local_hostname} going to remove {deleted_remote_hosts} from the websocket broadcast list")
+            if new_remote_hosts:
+                logger.warn(f"{self.local_hostname} going to add {new_remote_hosts} to the websocket broadcast list")
+
+            for h in deleted_remote_hosts:
+                self.broadcast_tasks[h].cancel()
+                del self.broadcast_tasks[h]
+                self.stats_mgr.delete_remote_host_stats(h)
+
+            for h in new_remote_hosts:
+                stats = self.stats_mgr.new_remote_host_stats(h)
+                broadcast_task = BroadcastWebsocketTask(name=self.local_hostname,
+                                                        event_loop=self.event_loop,
+                                                        stats=stats,
+                                                        remote_host=h)
+                broadcast_task.start()
+                self.broadcast_tasks[h] = broadcast_task
+
+            await asyncio.sleep(settings.BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS)
+
+    def start(self):
+        self.stats_mgr.start()
+
+        self.async_task = self.event_loop.create_task(self.run_per_host_websocket())
+        return self.async_task

awx/settings/defaults.py

@@ -421,8 +421,8 @@ os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199')
 
 BROKER_DURABILITY = True
 BROKER_POOL_LIMIT = None
-BROKER_URL = 'amqp://guest:guest@localhost:5672//'
-CELERY_DEFAULT_QUEUE = 'awx_private_queue'
+BROKER_URL = 'unix:///var/run/redis/redis.sock'
+BROKER_TRANSPORT_OPTIONS = {}
 CELERYBEAT_SCHEDULE = {
     'tower_scheduler': {
         'task': 'awx.main.tasks.awx_periodic_scheduler',
@@ -451,19 +451,6 @@ CELERYBEAT_SCHEDULE = {
     # 'isolated_heartbeat': set up at the end of production.py and development.py
 }
 
-AWX_CELERY_QUEUES_STATIC = [
-    CELERY_DEFAULT_QUEUE,
-]
-
-AWX_CELERY_BCAST_QUEUES_STATIC = [
-    'tower_broadcast_all',
-]
-
-ASGI_AMQP = {
-    'INIT_FUNC': 'awx.prepare_env',
-    'MODEL': 'awx.main.models.channels.ChannelGroup',
-}
-
 # Django Caching Configuration
 CACHES = {
     'default': {
@@ -929,8 +916,6 @@ ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC = False
 # Internal API URL for use by inventory scripts and callback plugin.
 INTERNAL_API_URL = 'http://127.0.0.1:%s' % DEVSERVER_DEFAULT_PORT
 
-PERSISTENT_CALLBACK_MESSAGES = True
-USE_CALLBACK_QUEUE = True
 CALLBACK_QUEUE = "callback_tasks"
 
 SCHEDULER_QUEUE = "scheduler"
@@ -965,6 +950,18 @@ LOG_AGGREGATOR_LEVEL = 'INFO'
 # raising this value can help
 CHANNEL_LAYER_RECEIVE_MAX_RETRY = 10
 
+ASGI_APPLICATION = "awx.main.routing.application"
+
+CHANNEL_LAYERS = {
+    "default": {
+        "BACKEND": "channels_redis.core.RedisChannelLayer",
+        "CONFIG": {
+            "hosts": [BROKER_URL],
+            "capacity": 10000,
+        },
+    },
+}
+
 # Logging configuration.
 LOGGING = {
     'version': 1,
@@ -1111,10 +1108,6 @@ LOGGING = {
             'handlers': ['console', 'file', 'tower_warnings'],
             'level': 'WARNING',
         },
-        'kombu': {
-            'handlers': ['console', 'file', 'tower_warnings'],
-            'level': 'WARNING',
-        },
         'rest_framework.request': {
             'handlers': ['console', 'file', 'tower_warnings'],
             'level': 'WARNING',
@@ -1239,3 +1232,29 @@ MIDDLEWARE = [
     'awx.main.middleware.URLModificationMiddleware',
     'awx.main.middleware.SessionTimeoutMiddleware',
 ]
+
+# Secret header value to exchange for websockets responsible for distributing websocket messages.
+# This needs to be kept secret and randomly generated
+BROADCAST_WEBSOCKET_SECRET = ''
+
+# Port for broadcast websockets to connect to
+# Note: that the clients will follow redirect responses
+BROADCAST_WEBSOCKET_PORT = 443
+
+# Whether or not broadcast websockets should check nginx certs when interconnecting
+BROADCAST_WEBSOCKET_VERIFY_CERT = False
+
+# Connect to other AWX nodes using http or https
+BROADCAST_WEBSOCKET_PROTOCOL = 'https'
+
+# All websockets that connect to the broadcast websocket endpoint will be put into this group
+BROADCAST_WEBSOCKET_GROUP_NAME = 'broadcast-group_send'
+
+# Time wait before retrying connecting to a websocket broadcast tower node
+BROADCAST_WEBSOCKET_RECONNECT_RETRY_RATE_SECONDS = 5
+
+# How often websocket process will look for changes in the Instance table
+BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS = 10
+
+# How often websocket process will generate stats
+BROADCAST_WEBSOCKET_STATS_POLL_RATE_SECONDS = 5

awx/settings/local_settings.py.docker_compose

@@ -12,7 +12,6 @@
 # MISC PROJECT SETTINGS
 ###############################################################################
 import os
-import urllib.parse
 import sys
 
 # Enable the following lines and install the browser extension to use Django debug toolbar
@@ -49,18 +48,6 @@ if "pytest" in sys.modules:
         }
     }
 
-# AMQP configuration.
-BROKER_URL = "amqp://{}:{}@{}/{}".format(os.environ.get("RABBITMQ_USER"),
-                                  os.environ.get("RABBITMQ_PASS"),
-                                  os.environ.get("RABBITMQ_HOST"),
-                                  urllib.parse.quote(os.environ.get("RABBITMQ_VHOST", "/"), safe=''))
-
-CHANNEL_LAYERS = {
-    'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',
-                'ROUTING': 'awx.main.routing.channel_routing',
-                'CONFIG': {'url': BROKER_URL}}
-}
-
 # Absolute filesystem path to the directory to host projects (with playbooks).
 # This directory should NOT be web-accessible.
 PROJECTS_ROOT = '/var/lib/awx/projects/'
@@ -238,3 +225,8 @@ TEST_OPENSTACK_PROJECT = ''
 # Azure credentials.
 TEST_AZURE_USERNAME = ''
 TEST_AZURE_KEY_DATA = ''
+
+BROADCAST_WEBSOCKET_SECRET = '🤖starscream🤖'
+BROADCAST_WEBSOCKET_PORT = 8013
+BROADCAST_WEBSOCKET_VERIFY_CERT = False
+BROADCAST_WEBSOCKET_PROTOCOL = 'http'

awxkit/awxkit/ws.py

@@ -1,4 +1,3 @@
-import time
 import threading
 import logging
 import atexit
@@ -93,6 +92,7 @@ class WSClient(object):
                                          cookie=auth_cookie)
         self._message_cache = []
         self._should_subscribe_to_pending_job = False
+        self._pending_unsubscribe = threading.Event()
 
     def connect(self):
         wst = threading.Thread(target=self._ws_run_forever, args=(self.ws, {"cert_reqs": ssl.CERT_NONE}))
@@ -184,11 +184,16 @@ class WSClient(object):
         payload['xrftoken'] = self.csrftoken
         self._send(json.dumps(payload))
 
-    def unsubscribe(self):
-        self._send(json.dumps(dict(groups={}, xrftoken=self.csrftoken)))
-        # it takes time for the unsubscribe event to be recieved and consumed and for
-        # messages to stop being put on the queue for daphne to send to us
-        time.sleep(5)
+    def unsubscribe(self, wait=True, timeout=10):
+        if wait:
+            # Other unnsubscribe events could have caused the edge to trigger.
+            # This way the _next_ event will trigger our waiting.
+            self._pending_unsubscribe.clear()
+            self._send(json.dumps(dict(groups={}, xrftoken=self.csrftoken)))
+            if not self._pending_unsubscribe.wait(timeout):
+                raise RuntimeError("Failed while waiting on unsubscribe reply because timeout of {} seconds was reached.".format(timeout))
+        else:
+            self._send(json.dumps(dict(groups={}, xrftoken=self.csrftoken)))
 
     def _on_message(self, message):
         message = json.loads(message)
@@ -202,7 +207,13 @@ class WSClient(object):
                     self._should_subscribe_to_pending_job['events'] == 'project_update_events'):
                 self._update_subscription(message['unified_job_id'])
 
-        return self._recv_queue.put(message)
+        ret = self._recv_queue.put(message)
+
+        # unsubscribe acknowledgement
+        if 'groups_current' in message:
+            self._pending_unsubscribe.set()
+
+        return ret
 
     def _update_subscription(self, job_id):
         subscription = dict(jobs=self._should_subscribe_to_pending_job['jobs'])

docs/licenses/aiohttp.txt

@@ -0,0 +1,201 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2013-2020 aiohttp maintainers
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

docs/licenses/aioredis.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014-2017 Alexey Popravka
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docs/licenses/asgi-amqp.txt

@@ -1,24 +0,0 @@
-# Copyright (c) 2017 Ansible by Red Hat
-# All Rights Reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in the
-      documentation and/or other materials provided with the distribution.
-    * Neither the name of the <organization> nor the
-      names of its contributors may be used to endorse or promote products
-      derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

docs/licenses/async-timeout.txt

@@ -1,18 +1,3 @@
-Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com>
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -193,7 +178,7 @@ Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com>
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
+      boilerplate notice, with the fields enclosed by brackets "{}"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -201,7 +186,7 @@ Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com>
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright {yyyy} {name of copyright owner}
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

docs/licenses/channels-redis.txt

@@ -0,0 +1,27 @@
+Copyright (c) Django Software Foundation and individual contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of Django nor the names of its contributors may be used
+       to endorse or promote products derived from this software without
+       specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

docs/licenses/hiredis.txt

@@ -0,0 +1 @@
+This code is released under the BSD license, after the license of hiredis.

docs/licenses/idna-ssl.txt

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2018 aio-libs team https://github.com/aio-libs/
+Copyright (c) 2017 Ocean S. A. https://ocean.io/
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

docs/licenses/inflect.txt

@@ -1,17 +0,0 @@
-Copyright (C) 2010 Paul Dyson
-
-Based upon the Perl module Lingua::EN::Inflect by Damian Conway.
-
-This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
-
-This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-The original Perl module Lingua::EN::Inflect by Damian Conway is available from http://search.cpan.org/~dconway/
-
-This module can be downloaded at http://pypi.python.org/pypi/inflect
-
-This module can be installed via easy_install inflect
-
-Repository available at http://github.com/pwdyson/inflect.py

docs/licenses/jaraco.itertools.txt

@@ -1,10 +0,0 @@
-# As listed on https://pypi.python.org/pypi/irc
-
-The MIT License (MIT)
-Copyright (c) <year> <copyright holders>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

docs/licenses/jsonpickle.txt

@@ -1,29 +0,0 @@
-Copyright (C) 2008 John Paulett (john -at- paulett.org)
-Copyright (C) 2009-2016 David Aguilar (davvid -at- gmail.com)
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
- 1. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in
-    the documentation and/or other materials provided with the
-    distribution.
- 3. The name of the author may not be used to endorse or promote
-    products derived from this software without specific prior
-    written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
-OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
-GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
-IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
-OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
-IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

docs/licenses/msgpack.txt

@@ -0,0 +1,13 @@
+Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com>
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

docs/licenses/multidict.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016-2017 Andrew Svetlov
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

docs/licenses/redis.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Andy McCurdy
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.

docs/licenses/service-identity.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2014 Hynek Schlawack
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docs/licenses/typing-extensions.txt

@@ -0,0 +1,254 @@
+A. HISTORY OF THE SOFTWARE
+==========================
+
+Python was created in the early 1990s by Guido van Rossum at Stichting
+Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
+as a successor of a language called ABC.  Guido remains Python's
+principal author, although it includes many contributions from others.
+
+In 1995, Guido continued his work on Python at the Corporation for
+National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
+in Reston, Virginia where he released several versions of the
+software.
+
+In May 2000, Guido and the Python core development team moved to
+BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
+year, the PythonLabs team moved to Digital Creations (now Zope
+Corporation, see http://www.zope.com).  In 2001, the Python Software
+Foundation (PSF, see http://www.python.org/psf/) was formed, a
+non-profit organization created specifically to own Python-related
+Intellectual Property.  Zope Corporation is a sponsoring member of
+the PSF.
+
+All Python releases are Open Source (see http://www.opensource.org for
+the Open Source Definition).  Historically, most, but not all, Python
+releases have also been GPL-compatible; the table below summarizes
+the various releases.
+
+    Release         Derived     Year        Owner       GPL-
+                    from                                compatible? (1)
+
+    0.9.0 thru 1.2              1991-1995   CWI         yes
+    1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
+    1.6             1.5.2       2000        CNRI        no
+    2.0             1.6         2000        BeOpen.com  no
+    1.6.1           1.6         2001        CNRI        yes (2)
+    2.1             2.0+1.6.1   2001        PSF         no
+    2.0.1           2.0+1.6.1   2001        PSF         yes
+    2.1.1           2.1+2.0.1   2001        PSF         yes
+    2.1.2           2.1.1       2002        PSF         yes
+    2.1.3           2.1.2       2002        PSF         yes
+    2.2 and above   2.1.1       2001-now    PSF         yes
+
+Footnotes:
+
+(1) GPL-compatible doesn't mean that we're distributing Python under
+    the GPL.  All Python licenses, unlike the GPL, let you distribute
+    a modified version without making your changes open source.  The
+    GPL-compatible licenses make it possible to combine Python with
+    other software that is released under the GPL; the others don't.
+
+(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
+    because its license has a choice of law clause.  According to
+    CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
+    is "not incompatible" with the GPL.
+
+Thanks to the many outside volunteers who have worked under Guido's
+direction to make these releases possible.
+
+
+B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
+===============================================================
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are
+retained in Python alone or in any derivative version prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
+-------------------------------------------
+
+BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
+
+1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
+office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
+Individual or Organization ("Licensee") accessing and otherwise using
+this software in source or binary form and its associated
+documentation ("the Software").
+
+2. Subject to the terms and conditions of this BeOpen Python License
+Agreement, BeOpen hereby grants Licensee a non-exclusive,
+royalty-free, world-wide license to reproduce, analyze, test, perform
+and/or display publicly, prepare derivative works, distribute, and
+otherwise use the Software alone or in any derivative version,
+provided, however, that the BeOpen Python License is retained in the
+Software, alone or in any derivative version prepared by Licensee.
+
+3. BeOpen is making the Software available to Licensee on an "AS IS"
+basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
+SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
+AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
+DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+5. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+6. This License Agreement shall be governed by and interpreted in all
+respects by the law of the State of California, excluding conflict of
+law provisions.  Nothing in this License Agreement shall be deemed to
+create any relationship of agency, partnership, or joint venture
+between BeOpen and Licensee.  This License Agreement does not grant
+permission to use BeOpen trademarks or trade names in a trademark
+sense to endorse or promote products or services of Licensee, or any
+third party.  As an exception, the "BeOpen Python" logos available at
+http://www.pythonlabs.com/logos.html may be used according to the
+permissions granted on that web page.
+
+7. By copying, installing or otherwise using the software, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
+---------------------------------------
+
+1. This LICENSE AGREEMENT is between the Corporation for National
+Research Initiatives, having an office at 1895 Preston White Drive,
+Reston, VA 20191 ("CNRI"), and the Individual or Organization
+("Licensee") accessing and otherwise using Python 1.6.1 software in
+source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, CNRI
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python 1.6.1
+alone or in any derivative version, provided, however, that CNRI's
+License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
+1995-2001 Corporation for National Research Initiatives; All Rights
+Reserved" are retained in Python 1.6.1 alone or in any derivative
+version prepared by Licensee.  Alternately, in lieu of CNRI's License
+Agreement, Licensee may substitute the following text (omitting the
+quotes): "Python 1.6.1 is made available subject to the terms and
+conditions in CNRI's License Agreement.  This Agreement together with
+Python 1.6.1 may be located on the Internet using the following
+unique, persistent identifier (known as a handle): 1895.22/1013.  This
+Agreement may also be obtained from a proxy server on the Internet
+using the following URL: http://hdl.handle.net/1895.22/1013".
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python 1.6.1 or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python 1.6.1.
+
+4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
+basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. This License Agreement shall be governed by the federal
+intellectual property law of the United States, including without
+limitation the federal copyright law, and, to the extent such
+U.S. federal law does not apply, by the law of the Commonwealth of
+Virginia, excluding Virginia's conflict of law provisions.
+Notwithstanding the foregoing, with regard to derivative works based
+on Python 1.6.1 that incorporate non-separable material that was
+previously distributed under the GNU General Public License (GPL), the
+law of the Commonwealth of Virginia shall govern this License
+Agreement only as to issues arising under or with respect to
+Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
+License Agreement shall be deemed to create any relationship of
+agency, partnership, or joint venture between CNRI and Licensee.  This
+License Agreement does not grant permission to use CNRI trademarks or
+trade name in a trademark sense to endorse or promote products or
+services of Licensee, or any third party.
+
+8. By clicking on the "ACCEPT" button where indicated, or by copying,
+installing or otherwise using Python 1.6.1, Licensee agrees to be
+bound by the terms and conditions of this License Agreement.
+
+        ACCEPT
+
+
+CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
+--------------------------------------------------
+
+Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
+The Netherlands.  All rights reserved.
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Stichting Mathematisch
+Centrum or CWI not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior
+permission.
+
+STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
+FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

docs/licenses/yarl.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016-2018, Andrew Svetlov and aio-libs team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

docs/websockets.md

@@ -6,7 +6,7 @@ Our channels/websocket implementation handles the communication between Tower AP
 
 Tower enlists the help of the `django-channels` library to create our communications layer. `django-channels` provides us with per-client messaging integration in our application by implementing the Asynchronous Server Gateway Interface (ASGI).
 
-To communicate between our different services we use RabbitMQ to exchange messages. Traditionally, `django-channels` uses Redis, but Tower uses a custom `asgi_amqp` library that allows access to RabbitMQ for the same purpose.
+To communicate between our different services we use websockets. Every AWX node is fully connected via a special websocket endpoint that forwards any local websocket data to all other nodes. Local websockets are backed by Redis, the channels2 default service.
 
 Inside Tower we use the `emit_channel_notification` function which places messages onto the queue. The messages are given an explicit event group and event type which we later use in our wire protocol to control message delivery to the client.
 
@@ -43,14 +43,14 @@ This section will specifically discuss deployment in the context of websockets a
 | `nginx`     | listens on ports 80/443, handles HTTPS proxying, serves static assets, routes requests for `daphne` and `uwsgi` |
 | `uwsgi`      | listens on port 8050, handles API requests |
 | `daphne`      | listens on port 8051, handles websocket requests |
-| `runworker`   | no listening port, watches and processes the message queue |
+| `wsbroadcast`   | no listening port, forwards all group messages to all cluster nodes |
 | `supervisord` | (production-only) handles the process management of all the services except `nginx` |
 
 When a request comes in to `nginx` and has the `Upgrade` header and is for the path `/websocket`, then `nginx` knows that it should be routing that request to our `daphne` service.
 
-`daphne` receives the request and generates channel and routing information for the request. The configured event handlers for `daphne` then unpack and parse the request message using the wire protocol mentioned above. This ensures that the connection has its context limited to only receive messages for events it is interested in. `daphne` uses internal events to trigger further behavior, which will generate messages and send them to the queue, which is then processed by the `runworker`.
+`daphne` handles websocket connections proxied by nginx.
 
-`runworker` processes the messages from the queue. This uses the contextual information of the message provided by the `daphne` server and our `asgi_amqp` implementation to broadcast messages out to each client.
+`wsbroadcast` fully connects all cluster nodes via the `/websocket/broadcast/` endpoint to every other cluster nodes. Sends a copy of all group websocket messages to all other cluster nodes (i.e. job event type messages).
 
 ### Development
  - `nginx` listens on 8013/8043 instead of 80/443

installer/inventory

@@ -38,7 +38,7 @@ dockerhub_base=ansible
 # kubernetes_ingress_tls_secret=awx-cert
 
 # Kubernetes and Openshift Install Resource Requests
-# These are the request and limit values for a pod's container for task/web/rabbitmq/memcached/management.
+# These are the request and limit values for a pod's container for task/web/redis/memcached/management.
 # The total amount of requested resources for a pod is the sum of all
 # resources requested by all containers in the pod
 # A cpu_request of 1500 is 1.5 cores for the container to start out with.
@@ -52,8 +52,8 @@ dockerhub_base=ansible
 # task_mem_limit=4
 # web_cpu_limit=1000
 # web_mem_limit=2
-# rabbitmq_cpu_limit=1000
-# rabbitmq_mem_limit=3
+# redis_cpu_limit=1000
+# redis_mem_limit=3
 # memcached_cpu_limit=1000
 # memcached_mem_limit=2
 # management_cpu_limit=2000
@@ -93,10 +93,6 @@ pg_port=5432
 # containerized postgres deployment on OpenShift
 # pg_admin_password=postgrespass
 
-# RabbitMQ Configuration
-rabbitmq_password=awxpass
-rabbitmq_erlang_cookie=cookiemonster
-
 # Use a local distribution build container image for building the AWX package
 # This is helpful if you don't want to bother installing the build-time dependencies as
 # it is taken care of already.

installer/roles/image_build/files/launch_awx.sh

@@ -9,7 +9,6 @@ source /etc/tower/conf.d/environment.sh
 
 ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all
 ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$MEMCACHED_HOST port=$MEMCACHED_PORT" all
-ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$RABBITMQ_HOST port=$RABBITMQ_PORT" all
 ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m postgresql_db --become-user $DATABASE_USER -a "name=$DATABASE_NAME owner=$DATABASE_USER login_user=$DATABASE_USER login_host=$DATABASE_HOST login_password=$DATABASE_PASSWORD port=$DATABASE_PORT" all
 
 awx-manage collectstatic --noinput --clear

installer/roles/image_build/files/settings.py

@@ -85,17 +85,4 @@ DATABASES = {
 if os.getenv("DATABASE_SSLMODE", False):
     DATABASES['default']['OPTIONS'] = {'sslmode': os.getenv("DATABASE_SSLMODE")}
 
-BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format(
-    os.getenv("RABBITMQ_USER", None),
-    os.getenv("RABBITMQ_PASSWORD", None),
-    os.getenv("RABBITMQ_HOST", None),
-    os.getenv("RABBITMQ_PORT", "5672"),
-    os.getenv("RABBITMQ_VHOST", "tower"))
-
-CHANNEL_LAYERS = {
-    'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',
-                'ROUTING': 'awx.main.routing.channel_routing',
-                'CONFIG': {'url': BROKER_URL}}
-}
-
 USE_X_FORWARDED_PORT = True

installer/roles/image_build/files/supervisor.conf

@@ -35,8 +35,19 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
+[program:wsbroadcast]
+command = awx-manage run_wsbroadcast
+directory = /var/lib/awx
+autostart = true
+autorestart = true
+stopwaitsecs = 5
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
 [group:tower-processes]
-programs=nginx,uwsgi,daphne
+programs=nginx,uwsgi,daphne,wsbroadcast
 priority=5
 
 # TODO: Exit Handler

installer/roles/image_build/files/supervisor_task.conf

@@ -26,19 +26,8 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
-[program:channels-worker]
-command = awx-manage runworker --only-channels websocket.*
-directory = /var/lib/awx
-autostart = true
-autorestart = true
-stopwaitsecs = 5
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-
 [group:tower-processes]
-programs=dispatcher,callback-receiver,channels-worker
+programs=dispatcher,callback-receiver
 priority=5
 
 # TODO: Exit Handler

installer/roles/image_build/templates/launch_awx_task.sh.j2

@@ -9,7 +9,6 @@ source /etc/tower/conf.d/environment.sh
 
 ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all
 ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$MEMCACHED_HOST port=$MEMCACHED_PORT" all
-ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$RABBITMQ_HOST port=$RABBITMQ_PORT" all
 ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m postgresql_db --become-user $DATABASE_USER -a "name=$DATABASE_NAME owner=$DATABASE_USER login_user=$DATABASE_USER login_host=$DATABASE_HOST login_password=$DATABASE_PASSWORD port=$DATABASE_PORT" all
 
 if [ -z "$AWX_SKIP_MIGRATIONS" ]; then

installer/roles/kubernetes/defaults/main.yml

@@ -6,10 +6,6 @@ admin_user: 'admin'
 admin_email: 'root@localhost'
 admin_password: ''
 
-rabbitmq_user: 'awx'
-rabbitmq_password: ''
-rabbitmq_erlang_cookie: ''
-
 kubernetes_base_path: "{{ local_base_config_path|default('/tmp') }}/{{ kubernetes_deployment_name }}-config"
 
 kubernetes_task_version: "{{ tower_package_version | default(dockerhub_version) }}"
@@ -24,16 +20,17 @@ web_cpu_request: 500
 task_mem_request: 2
 task_cpu_request: 1500
 
-rabbitmq_mem_request: 2
-rabbitmq_cpu_request: 500
+redis_mem_request: 2
+redis_cpu_request: 500
+
+kubernetes_redis_image: "redis"
+kubernetes_redis_image_tag: "latest"
+kubernetes_redis_config_mount_path: "/usr/local/etc/redis/redis.conf"
 
 memcached_hostname: localhost
 memcached_mem_request: 1
 memcached_cpu_request: 500
 
-kubernetes_rabbitmq_version: "3.7.21"
-kubernetes_rabbitmq_image: "ansible/awx_rabbitmq"
-
 kubernetes_memcached_version: "latest"
 kubernetes_memcached_image: "memcached"
 
@@ -56,6 +53,5 @@ custom_venvs_path: "/opt/custom-venvs"
 custom_venvs_python: "python2"
 
 ca_trust_bundle: "/etc/pki/tls/certs/ca-bundle.crt"
-rabbitmq_use_ssl: false
 
 container_groups_image: "ansible/ansible-runner"

installer/roles/kubernetes/tasks/main.yml

@@ -194,10 +194,6 @@
       when: kubernetes_web_image is not defined
   when: docker_registry is defined
 
-- name: Generate SSL certificates for RabbitMQ, if needed
-  include_tasks: ssl_cert_gen.yml
-  when: "rabbitmq_use_ssl|default(False)|bool"
-
 - name: Determine StatefulSet api version
   set_fact:
     kubernetes_statefulset_api_version: "{{ 'apps/v1' if kube_api_version is version('1.9', '>=') else 'apps/v1beta1' }}"

installer/roles/kubernetes/tasks/ssl_cert_gen.yml

@@ -1,60 +0,0 @@
----
-
-- name: Create temporary directory
-  tempfile:
-    state: directory
-    prefix: "tower-install-rmq-certs"
-  register: rmq_cert_tempdir
-  notify: remove-rmq_cert_tempdir
-
-- name: Generate CA private key
-  openssl_privatekey:
-    path: '{{ rmq_cert_tempdir.path }}/ca.key'
-    mode: "0600"
-
-- name: Generate CA CSR
-  openssl_csr:
-    path: '{{ rmq_cert_tempdir.path }}/ca.csr'
-    privatekey_path: '{{ rmq_cert_tempdir.path }}/ca.key'
-    common_name: 'rabbitmq-ca'
-    basic_constraints: 'CA:TRUE'
-    mode: "0600"
-
-- name: Generate CA certificate
-  openssl_certificate:
-    path: '{{ rmq_cert_tempdir.path }}/ca.crt'
-    csr_path: '{{ rmq_cert_tempdir.path }}/ca.csr'
-    privatekey_path: '{{ rmq_cert_tempdir.path }}/ca.key'
-    provider: selfsigned
-    selfsigned_not_after: "+36524d"
-    mode: "0600"
-
-- name: Generate server private key
-  openssl_privatekey:
-    path: '{{ rmq_cert_tempdir.path }}/server.key'
-    mode: "0600"
-
-- name: Generate server CSR
-  openssl_csr:
-    path: '{{ rmq_cert_tempdir.path }}/server.csr'
-    privatekey_path: '{{ rmq_cert_tempdir.path }}/server.key'
-    common_name: 'rabbitmq-server'
-    mode: "0600"
-
-- name: Generate server certificate
-  openssl_certificate:
-    path: "{{ rmq_cert_tempdir.path }}/server.crt"
-    csr_path: "{{ rmq_cert_tempdir.path }}/server.csr"
-    privatekey_path: "{{ rmq_cert_tempdir.path }}/server.key"
-    provider: ownca
-    ownca_path: "{{ rmq_cert_tempdir.path }}/ca.crt"
-    ownca_privatekey_path: "{{ rmq_cert_tempdir.path }}/ca.key"
-    ownca_not_after: "+36500d"
-    mode: "0600"
-
-- name: Create combined certificate
-  assemble:
-    src: "{{ rmq_cert_tempdir.path }}"
-    regexp: "server.crt|server.key"
-    dest: "{{ rmq_cert_tempdir.path }}/server-combined.pem"
-    mode: "0600"

installer/roles/kubernetes/templates/configmap.yml.j2

@@ -205,3 +205,11 @@ data:
     USE_X_FORWARDED_PORT = True
 
     AWX_CONTAINER_GROUP_DEFAULT_IMAGE = "{{ container_groups_image }}"
+    BROADCAST_WEBSOCKET_PORT = 8052
+    BROADCAST_WEBSOCKET_PROTOCOL = 'http'
+
+  {{ kubernetes_deployment_name }}_redis_conf: |
+    unixsocket /var/run/redis/redis.sock
+    unixsocketperm 777
+    port 0
+    bind 127.0.0.1

installer/roles/kubernetes/templates/credentials.py.j2

@@ -12,14 +12,3 @@ DATABASES = {
         },
     }
 }
-BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format(
-    "{{ rabbitmq_user }}",
-    "{{ rabbitmq_password }}",
-    "localhost",
-    "5672",
-    "awx")
-CHANNEL_LAYERS = {
-    'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',
-                'ROUTING': 'awx.main.routing.channel_routing',
-                'CONFIG': {'url': BROKER_URL}}
-}

installer/roles/kubernetes/templates/deployment.yml.j2

@@ -15,131 +15,6 @@ imagePullSecrets:
   - name: "{{ kubernetes_image_pull_secrets }}"
 {% endif %}
 
----
-kind: Service
-apiVersion: v1
-metadata:
-  namespace: {{ kubernetes_namespace }}
-  name: rabbitmq
-  labels:
-    app: {{ kubernetes_deployment_name }}
-    type: LoadBalancer
-spec:
-  type: NodePort
-  ports:
-   - name: http
-     protocol: TCP
-     port: 15672
-     targetPort: 15672
-   - name: amqp
-     protocol: TCP
-     port: 5672
-     targetPort: 5672
-  selector:
-    app: {{ kubernetes_deployment_name }}
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: rabbitmq-config
-  namespace: {{ kubernetes_namespace }}
-data:
-  enabled_plugins: |
-      [rabbitmq_management,rabbitmq_peer_discovery_k8s].
-  rabbitmq_definitions.json: |
-      {
-        "users":[{"name": "{{ rabbitmq_user }}", "password": "{{ rabbitmq_password }}", "tags": "administrator"}],
-        "permissions":[
-          {"user":"{{ rabbitmq_user }}","vhost":"awx","configure":".*","write":".*","read":".*"}
-        ],
-        "vhosts":[{"name":"awx"}],
-        "policies":[
-            {"vhost":"awx","name":"ha-all","pattern":".*","definition":{"ha-mode":"all","ha-sync-mode":"automatic"}}
-        ]
-      }
-  rabbitmq.conf: |
-      ## Clustering
-      management.load_definitions = /etc/rabbitmq/rabbitmq_definitions.json
-      cluster_formation.peer_discovery_backend  = rabbit_peer_discovery_k8s
-      cluster_formation.k8s.host = kubernetes.default.svc
-      cluster_formation.k8s.address_type = ip
-      cluster_formation.node_cleanup.interval = 10
-      cluster_formation.node_cleanup.only_log_warning = false
-      cluster_partition_handling = autoheal
-      ## queue master locator
-      queue_master_locator=min-masters
-      ## enable guest user
-      loopback_users.guest = false
-{% if rabbitmq_use_ssl|default(False)|bool %}
-      ssl_options.cacertfile=/etc/pki/rabbitmq/ca.crt
-      ssl_options.certfile=/etc/pki/rabbitmq/server-combined.pem
-      ssl_options.verify=verify_peer
-{% endif %}
-  rabbitmq-env.conf: |
-      NODENAME=${RABBITMQ_NODENAME}
-      USE_LONGNAME=true
-{% if rabbitmq_use_ssl|default(False)|bool %}
-      ERL_SSL_PATH=$(erl -eval 'io:format("~p", [code:lib_dir(ssl, ebin)]),halt().' -noshell)
-      SSL_ADDITIONAL_ERL_ARGS="-pa '$ERL_SSL_PATH' -proto_dist inet_tls -ssl_dist_opt server_certfile /etc/pki/rabbitmq/server-combined.pem -ssl_dist_opt server_secure_renegotiate true client_secure_renegotiate true"
-      SERVER_ADDITIONAL_ERL_ARGS="$SERVER_ADDITIONAL_ERL_ARGS $SSL_ADDITIONAL_ERL_ARGS"
-      CTL_ERL_ARGS="$SSL_ADDITIONAL_ERL_ARGS"
-{% endif %}
-
-{% if kubernetes_context is defined %}
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: endpoint-reader
-  namespace: {{ kubernetes_namespace }}
-rules:
-- apiGroups: [""]
-  resources: ["endpoints"]
-  verbs: ["get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: endpoint-reader
-  namespace: {{ kubernetes_namespace }}
-subjects:
-- kind: ServiceAccount
-  name: awx
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: endpoint-reader
-{% endif %}
-
-{% if openshift_host is defined %}
----
-kind: Role
-apiVersion: v1
-metadata:
-  name: endpoint-reader
-  namespace: {{ kubernetes_namespace }}
-rules:
-  - apiGroups: [""]
-    resources: ["endpoints"]
-    verbs: ["get"]
----
-kind: RoleBinding
-apiVersion: v1
-metadata:
-  name: endpoint-reader
-  namespace: {{ kubernetes_namespace }}
-roleRef:
-  name: endpoint-reader
-  namespace: {{ kubernetes_namespace }}
-subjects:
-  - kind: ServiceAccount
-    name: awx
-    namespace: {{ kubernetes_namespace }}
-userNames:
-  - system:serviceaccount:{{ kubernetes_namespace }}:awx
-{% endif %}
-
 ---
 apiVersion: {{ kubernetes_statefulset_api_version }}
 kind: StatefulSet
@@ -253,6 +128,9 @@ spec:
               subPath: SECRET_KEY
               readOnly: true
 
+            - name: {{ kubernetes_deployment_name }}-redis-socket
+              mountPath: "/var/run/redis"
+
           resources:
             requests:
               memory: "{{ web_mem_request }}Gi"
@@ -266,7 +144,7 @@ spec:
 {% if web_cpu_limit is defined %}
               cpu: "{{ web_cpu_limit }}m"
 {% endif %}
-        - name: {{ kubernetes_deployment_name }}-celery
+        - name: {{ kubernetes_deployment_name }}-task
           securityContext:
             privileged: true
           image: "{{ kubernetes_task_image }}:{{ kubernetes_task_version }}"
@@ -296,6 +174,9 @@ spec:
               mountPath: "/etc/tower/SECRET_KEY"
               subPath: SECRET_KEY
               readOnly: true
+
+            - name: {{ kubernetes_deployment_name }}-redis-socket
+              mountPath: "/var/run/redis"
           env:
             - name: AWX_SKIP_MIGRATIONS
               value: "1"
@@ -303,6 +184,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.uid
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
           resources:
             requests:
               memory: "{{ task_mem_request }}Gi"
@@ -316,72 +201,34 @@ spec:
 {% if task_cpu_limit is defined %}
               cpu: "{{ task_cpu_limit }}m"
 {% endif %}
-        - name: {{ kubernetes_deployment_name }}-rabbit
-          image: "{{ kubernetes_rabbitmq_image }}:{{ kubernetes_rabbitmq_version }}"
+        - name: {{ kubernetes_deployment_name }}-redis
+          image: {{ kubernetes_redis_image }}:{{ kubernetes_redis_image_tag }}
           imagePullPolicy: Always
+          args: ["/usr/local/etc/redis/redis.conf"]
           ports:
-            - name: http
+            - name: redis
               protocol: TCP
-              containerPort: 15672
-            - name: amqp
-              protocol: TCP
-              containerPort: 5672
-          livenessProbe:
-            exec:
-              command:
-              - /usr/local/bin/healthchecks/rabbit_health_node.py
-            initialDelaySeconds: 30
-            timeoutSeconds: 10
-          readinessProbe:
-            exec:
-              command:
-              - /usr/local/bin/healthchecks/rabbit_health_node.py
-            initialDelaySeconds: 10
-            timeoutSeconds: 10
-          env:
-            - name: MY_POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: RABBITMQ_USE_LONGNAME
-              value: "true"
-            - name: RABBITMQ_NODENAME
-              value: "rabbit@$(MY_POD_IP)"
-            - name: RABBITMQ_ERLANG_COOKIE
-              valueFrom:
-                secretKeyRef:
-                  name: "{{ kubernetes_deployment_name }}-secrets"
-                  key: rabbitmq_erlang_cookie
-            - name: K8S_SERVICE_NAME
-              value: "rabbitmq"
-            - name: RABBITMQ_USER
-              value: {{ rabbitmq_user }}
-            - name: RABBITMQ_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: "{{ kubernetes_deployment_name }}-secrets"
-                  key: rabbitmq_password
+              containerPort: 6379
           volumeMounts:
-            - name: rabbitmq-config
-              mountPath: /etc/rabbitmq
-            - name: rabbitmq-healthchecks
-              mountPath: /usr/local/bin/healthchecks
-{% if rabbitmq_use_ssl|default(False)|bool %}
-            - name: "{{ kubernetes_deployment_name }}-rabbitmq-certs-vol"
-              mountPath: /etc/pki/rabbitmq
-{% endif %}
+            - name: {{ kubernetes_deployment_name }}-redis-config
+              mountPath: "{{ kubernetes_redis_config_mount_path }}"
+              subPath: redis.conf
+              readOnly: true
+
+            - name: {{ kubernetes_deployment_name }}-redis-socket
+              mountPath: "/var/run/redis"
           resources:
             requests:
-              memory: "{{ rabbitmq_mem_request }}Gi"
-              cpu: "{{ rabbitmq_cpu_request }}m"
-{% if rabbitmq_mem_limit is defined or rabbitmq_cpu_limit is defined %}
+              memory: "{{ redis_mem_request }}Gi"
+              cpu: "{{ redis_cpu_request }}m"
+{% if redis_mem_limit is defined or redis_cpu_limit is defined %}
             limits:
 {% endif %}
-{% if rabbitmq_mem_limit is defined %}
-              memory: "{{ rabbitmq_mem_limit }}Gi"
+{% if redis_mem_limit is defined %}
+              memory: "{{ redis_mem_limit }}Gi"
 {% endif %}
-{% if rabbitmq_cpu_limit is defined %}
-              cpu: "{{ rabbitmq_cpu_limit }}m"
+{% if redis_cpu_limit is defined %}
+              cpu: "{{ redis_cpu_limit }}m"
 {% endif %}
         - name: {{ kubernetes_deployment_name }}-memcached
           image: "{{ kubernetes_memcached_image }}:{{ kubernetes_memcached_version }}"
@@ -442,6 +289,13 @@ spec:
               - key: {{ kubernetes_deployment_name }}_nginx_conf
                 path: nginx.conf
 
+        - name: {{ kubernetes_deployment_name }}-redis-config
+          configMap:
+            name: {{ kubernetes_deployment_name }}-config
+            items:
+              - key: {{ kubernetes_deployment_name }}_redis_conf
+                path: redis.conf
+
         - name: "{{ kubernetes_deployment_name }}-application-credentials"
           secret:
             secretName: "{{ kubernetes_deployment_name }}-secrets"
@@ -458,68 +312,9 @@ spec:
               - key: secret_key
                 path: SECRET_KEY
 
-        - name: rabbitmq-config
-          configMap:
-            name: rabbitmq-config
-            items:
-            - key: rabbitmq.conf
-              path: rabbitmq.conf
-            - key: enabled_plugins
-              path: enabled_plugins
-            - key: rabbitmq_definitions.json
-              path: rabbitmq_definitions.json
-            - key: rabbitmq-env.conf
-              path: rabbitmq-env.conf
+        - name: {{ kubernetes_deployment_name }}-redis-socket
+          emptyDir: {}
 
-{% if rabbitmq_use_ssl|default(False)|bool %}
-        - name: "{{ kubernetes_deployment_name }}-rabbitmq-certs-vol"
-          secret:
-            secretName: "{{ kubernetes_deployment_name }}-rabbitmq-certs"
-            items:
-              - key: rabbitmq_ssl_cert
-                path: 'server.crt'
-              - key: rabbitmq_ssl_key
-                path: 'server.key'
-              - key: rabbitmq_ssl_cacert
-                path: 'ca.crt'
-              - key: rabbitmq_ssl_combined
-                path: 'server-combined.pem'
-{% endif %}
-        - name: rabbitmq-healthchecks
-          configMap:
-            name: {{ kubernetes_deployment_name }}-healthchecks
-            items:
-            - key: rabbit_health_node.py
-              path: rabbit_health_node.py
-            defaultMode: 0755
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ kubernetes_deployment_name }}-healthchecks
-  namespace: {{ kubernetes_namespace }}
-data:
-  rabbit_health_node.py: |
-    #!/usr/bin/env python
-    try:
-      from http.client import HTTPConnection
-    except ImportError:
-      from httplib import HTTPConnection
-    import sys
-    import os
-    import base64
-    authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD'))
-    conn=HTTPConnection('localhost:15672')
-    conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret})
-    r1 = conn.getresponse()
-    if r1.status != 200:
-      sys.stderr.write('Received http error %i\\n' % (r1.status))
-      sys.exit(1)
-    body = r1.read()
-    if body != '{"status":"ok"}':
-      sys.stderr.write('Received body: %s' % body)
-      sys.exit(2)
-    sys.exit(0)
 ---
 apiVersion: v1
 kind: Service
@@ -536,22 +331,7 @@ spec:
       targetPort: 8052
   selector:
     name: {{ kubernetes_deployment_name }}-web-deploy
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ kubernetes_deployment_name }}-rmq-mgmt
-  namespace: {{ kubernetes_namespace }}
-  labels:
-    name: {{ kubernetes_deployment_name }}-rmq-mgmt
-spec:
-  type: ClusterIP
-  ports:
-    - name: rmqmgmt
-      port: 15672
-      targetPort: 15672
-  selector:
-    name: {{ kubernetes_deployment_name }}-web-deploy
+
 {% if kubernetes_context is defined %}
 ---
 apiVersion: extensions/v1beta1

installer/roles/kubernetes/templates/environment.sh.j2

@@ -5,5 +5,3 @@ DATABASE_PORT={{ pg_port|default('5432') }}
 DATABASE_PASSWORD={{ pg_password | quote }}
 MEMCACHED_HOST={{ memcached_hostname|default('localhost') }}
 MEMCACHED_PORT={{ memcached_port|default('11211') }}
-RABBITMQ_HOST={{ rabbitmq_hostname|default('localhost') }}
-RABBITMQ_PORT={{ rabbitmq_port|default('5672') }}

installer/roles/kubernetes/templates/management-pod.yml.j2

@@ -12,6 +12,7 @@ spec:
   containers:
     - name: ansible-tower-management
       image: "{{ kubernetes_task_image }}:{{ kubernetes_task_version }}"
+      imagePullPolicy: Always
       command: ["sleep", "infinity"]
       volumeMounts:
         - name: {{ kubernetes_deployment_name }}-application-config

installer/roles/kubernetes/templates/secret.yml.j2

@@ -7,22 +7,5 @@ metadata:
 type: Opaque
 data:
   secret_key: "{{ secret_key | b64encode }}"
-  rabbitmq_password: "{{ rabbitmq_password | b64encode }}"
-  rabbitmq_erlang_cookie: "{{ rabbitmq_erlang_cookie | b64encode }}"
   credentials_py: "{{ lookup('template', 'credentials.py.j2') | b64encode }}"
   environment_sh: "{{ lookup('template', 'environment.sh.j2') | b64encode }}"
-
-{% if rabbitmq_use_ssl|default(False)|bool %}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  namespace: {{ kubernetes_namespace }}
-  name: "{{ kubernetes_deployment_name }}-rabbitmq-certs"
-type: Opaque
-data:
-  rabbitmq_ssl_cert: "{{ lookup('file', rmq_cert_tempdir.path + '/server.crt') | b64encode }}"
-  rabbitmq_ssl_key: "{{ lookup('file', rmq_cert_tempdir.path + '/server.key') | b64encode }}"
-  rabbitmq_ssl_cacert: "{{ lookup('file', rmq_cert_tempdir.path + '/ca.crt') | b64encode }}"
-  rabbitmq_ssl_combined: "{{ lookup('file', rmq_cert_tempdir.path + '/server-combined.pem') | b64encode }}"
-{% endif %}

installer/roles/local_docker/defaults/main.yml

@@ -1,19 +1,11 @@
 ---
 dockerhub_version: "{{ lookup('file', playbook_dir + '/../VERSION') }}"
 
-rabbitmq_version: "3.7.4"
-rabbitmq_image: "ansible/awx_rabbitmq:{{rabbitmq_version}}"
-rabbitmq_default_vhost: "awx"
-rabbitmq_erlang_cookie: "cookiemonster"
-rabbitmq_hostname: "rabbitmq"
-rabbitmq_port: "5672"
-rabbitmq_user: "guest"
-rabbitmq_password: "guest"
+redis_image: "redis"
 
 postgresql_version: "10"
 postgresql_image: "postgres:{{postgresql_version}}"
 
-
 memcached_image: "memcached"
 memcached_version: "alpine"
 memcached_hostname: "memcached"

installer/roles/local_docker/tasks/compose.yml

@@ -4,6 +4,12 @@
     path: "{{ docker_compose_dir }}"
     state: directory
 
+- name: Create Redis socket directory
+  file:
+    path: "{{ docker_compose_dir }}/redis_socket"
+    state: directory
+    mode: 0777
+
 - name: Create Docker Compose Configuration
   template:
     src: "{{ item }}.j2"
@@ -14,8 +20,14 @@
     - credentials.py
     - docker-compose.yml
     - nginx.conf
+    - redis.conf
   register: awx_compose_config
 
+- name: Set redis config to other group readable to satisfy redis-server
+  file:
+    path: "{{ docker_compose_dir }}/redis.conf"
+    mode: 0666
+
 - name: Render SECRET_KEY file
   copy:
     content: "{{ secret_key }}"

installer/roles/local_docker/templates/credentials.py.j2

@@ -10,19 +10,6 @@ DATABASES = {
     }
 }
 
-BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format(
-    "{{ rabbitmq_user }}",
-    "{{ rabbitmq_password }}",
-    "{{ rabbitmq_hostname | default('rabbitmq')}}",
-    "{{ rabbitmq_port }}",
-    "{{ rabbitmq_default_vhost }}")
-
-CHANNEL_LAYERS = {
-    'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',
-                'ROUTING': 'awx.main.routing.channel_routing',
-                'CONFIG': {'url': BROKER_URL}}
-}
-
 CACHES = {
     'default': {
         'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',

installer/roles/local_docker/templates/docker-compose.yml.j2

@@ -6,7 +6,7 @@ services:
     image: {{ awx_web_docker_actual_image }}
     container_name: awx_web
     depends_on:
-      - rabbitmq
+      - redis
       - memcached
       {% if pg_hostname is not defined %}
       - postgres
@@ -24,6 +24,7 @@ services:
       - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh"
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
       - "{{ docker_compose_dir }}/nginx.conf:/etc/nginx/nginx.conf:ro"
+      - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw"
     {% if project_data_dir is defined %}
       - "{{ project_data_dir +':/var/lib/awx/projects:rw' }}"
     {% endif %}
@@ -63,7 +64,7 @@ services:
     image: {{ awx_task_docker_actual_image }}
     container_name: awx_task
     depends_on:
-      - rabbitmq
+      - redis
       - memcached
       - web
       {% if pg_hostname is not defined %}
@@ -76,6 +77,7 @@ services:
       - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY"
       - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh"
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
+      - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw"
     {% if project_data_dir is defined %}
       - "{{ project_data_dir +':/var/lib/awx/projects:rw' }}"
     {% endif %}
@@ -111,18 +113,18 @@ services:
       https_proxy: {{ https_proxy | default('') }}
       no_proxy: {{ no_proxy | default('') }}
 
-  rabbitmq:
-    image: {{ rabbitmq_image }}
-    container_name: awx_rabbitmq
+  redis:
+    image: {{ redis_image }}
+    container_name: awx_redis
     restart: unless-stopped
     environment:
-      RABBITMQ_DEFAULT_VHOST: "{{ rabbitmq_default_vhost }}"
-      RABBITMQ_DEFAULT_USER: "{{ rabbitmq_user }}"
-      RABBITMQ_DEFAULT_PASS: "{{ rabbitmq_password | quote }}"
-      RABBITMQ_ERLANG_COOKIE: {{ rabbitmq_erlang_cookie }}
       http_proxy: {{ http_proxy | default('') }}
       https_proxy: {{ https_proxy | default('') }}
       no_proxy: {{ no_proxy | default('') }}
+    command: ["/usr/local/etc/redis/redis.conf"]
+    volumes:
+      - "{{ docker_compose_dir }}/redis.conf:/usr/local/etc/redis/redis.conf:ro"
+      - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw"
 
   memcached:
     image: "{{ memcached_image }}:{{ memcached_version }}"

installer/roles/local_docker/templates/environment.sh.j2

@@ -8,7 +8,5 @@ DATABASE_ADMIN_PASSWORD={{ pg_admin_password|quote }}
 {% endif %}
 MEMCACHED_HOST={{ memcached_hostname|default('memcached') }}
 MEMCACHED_PORT={{ memcached_port|default('11211')|quote }}
-RABBITMQ_HOST={{ rabbitmq_hostname|default('rabbitmq')|quote }}
-RABBITMQ_PORT={{ rabbitmq_port|default('5672')|quote }}
 AWX_ADMIN_USER={{ admin_user|quote }}
 AWX_ADMIN_PASSWORD={{ admin_password|quote }}

installer/roles/local_docker/templates/redis.conf.j2

@@ -0,0 +1,4 @@
+unixsocket /var/run/redis/redis.sock
+unixsocketperm 777
+port 0
+bind 127.0.0.1

requirements/README.md

@@ -144,8 +144,3 @@ in the top-level Makefile.
 
 This can be removed when a solution for the external log queuing is ready.
 https://github.com/ansible/awx/pull/5092
-
-### asgi-amqp
-
-This library is not compatible with channels 2 and is not expected
-to become so. This drives other pins in the requirements file.

requirements/requirements.in

@@ -1,10 +1,11 @@
+aiohttp
 ansible-runner
 ansiconv==1.0.0  # UPGRADE BLOCKER: from 2013, consider replacing instead of upgrading
-asgi-amqp>=1.1.4  # see library notes, related to channels 2
 azure-keyvault==1.1.0  # see UPGRADE BLOCKERs
 boto  # replacement candidate https://github.com/ansible/awx/issues/2115
-channels==1.1.8  # UPGRADE BLOCKER: Last before backwards-incompatible channels 2 upgrade
-daphne==1.4.2  # UPGRADE BLOCKER: last before channels 2 but not pinned by other deps
+channels
+channels-redis
+daphne
 django==2.2.10  # see UPGRADE BLOCKERs
 django-auth-ldap
 django-cors-headers
@@ -37,6 +38,7 @@ python3-saml
 schedule==0.6.0
 social-auth-core==3.2.0  # see UPGRADE BLOCKERs
 social-auth-app-django==3.1.0  # see UPGRADE BLOCKERs
+redis
 requests
 requests-futures  # see library notes
 slackclient==1.1.2  # see UPGRADE BLOCKERs

requirements/requirements.txt

@@ -1,33 +1,36 @@
 adal==1.2.2               # via msrestazure
+aiohttp==3.6.2
+aioredis==1.3.1           # via channels-redis
 amqp==2.5.2               # via kombu
 ansible-runner==1.4.4
 ansiconv==1.0.0
-asgi-amqp==1.1.4
-asgiref==1.1.2            # via asgi-amqp, channels, daphne
-attrs==19.3.0             # via automat, jsonschema, twisted
-autobahn==19.11.1         # via daphne
+asgiref==3.2.3            # via channels, channels-redis, daphne
+async-timeout==3.0.1      # via aiohttp, aioredis
+attrs==19.3.0             # via aiohttp, automat, jsonschema, service-identity, twisted
+autobahn==20.1.2          # via daphne
 automat==0.8.0            # via twisted
-azure-common==1.1.23      # via azure-keyvault
+azure-common==1.1.24      # via azure-keyvault
 azure-keyvault==1.1.0
 azure-nspkg==3.0.2        # via azure-keyvault
 boto==2.49.0
-cachetools==3.1.1         # via google-auth
+cachetools==4.0.0         # via google-auth
 certifi==2019.11.28       # via kubernetes, msrest, requests
 cffi==1.13.2              # via cryptography
-channels==1.1.8
-chardet==3.0.4            # via requests
+channels-redis==2.4.1
+channels==2.4.0
+chardet==3.0.4            # via aiohttp, requests
 constantly==15.1.0        # via twisted
-cryptography==2.8         # via adal, autobahn, azure-keyvault
-daphne==1.4.2
+cryptography==2.8         # via adal, autobahn, azure-keyvault, pyopenssl, service-identity
+daphne==2.4.1
 defusedxml==0.6.0         # via python3-openid, python3-saml, social-auth-core
 dictdiffer==0.8.1         # via openshift
 django-auth-ldap==2.1.0
-django-cors-headers==3.2.0
+django-cors-headers==3.2.1
 django-crum==0.7.5
 django-extensions==2.2.5
 django-jsonfield==1.2.0
 django-oauth-toolkit==1.1.3
-django-pglocks==1.0.3
+django-pglocks==1.0.4
 django-polymorphic==2.1.2
 django-qsstats-magic==1.1.0
 django-radius==1.3.3
@@ -41,24 +44,23 @@ docutils==0.15.2          # via python-daemon
 future==0.16.0            # via django-radius
 gitdb2==2.0.6             # via gitpython
 gitpython==3.0.5
-google-auth==1.9.0        # via kubernetes
+google-auth==1.10.0       # via kubernetes
+hiredis==1.0.1            # via aioredis
 hyperlink==19.0.0         # via twisted
-idna==2.8                 # via hyperlink, requests
-importlib-metadata==1.3.0  # via inflect, irc, jsonschema, kombu
+idna-ssl==1.1.0           # via aiohttp
+idna==2.8                 # via hyperlink, idna-ssl, requests, twisted, yarl
+importlib-metadata==1.4.0  # via irc, jsonschema, kombu
 importlib-resources==1.0.2  # via jaraco.text
 incremental==17.5.0       # via twisted
-inflect==3.0.2            # via jaraco.itertools
-irc==17.1
+irc==18.0.0
 isodate==0.6.0            # via msrest, python3-saml
-jaraco.classes==2.0       # via jaraco.collections
-jaraco.collections==2.1   # via irc
-jaraco.functools==2.0     # via irc, jaraco.text, tempora
-jaraco.itertools==4.4.2   # via irc
-jaraco.logging==2.0       # via irc
+jaraco.classes==3.1.0     # via jaraco.collections
+jaraco.collections==3.0.0  # via irc
+jaraco.functools==3.0.0   # via irc, jaraco.text, tempora
+jaraco.logging==3.0.0     # via irc
 jaraco.stream==3.0.0      # via irc
 jaraco.text==3.2.0        # via irc, jaraco.collections
 jinja2==2.10.3
-jsonpickle==1.2           # via asgi-amqp
 jsonschema==3.2.0
 kombu==4.6.7              # via asgi-amqp
 kubernetes==10.0.1        # via openshift
@@ -66,10 +68,11 @@ lockfile==0.12.2          # via python-daemon
 lxml==4.4.2               # via xmlsec
 markdown==3.1.1
 markupsafe==1.1.1         # via jinja2
-more-itertools==8.0.2     # via irc, jaraco.functools, jaraco.itertools, zipp
-msgpack-python==0.5.6     # via asgi-amqp
+more-itertools==8.1.0     # via irc, jaraco.classes, jaraco.functools, zipp
+msgpack==0.6.2            # via channels-redis
 msrest==0.6.10            # via azure-keyvault, msrestazure
 msrestazure==0.6.2        # via azure-keyvault
+multidict==4.7.4          # via aiohttp, yarl
 netaddr==0.7.19           # via pyrad
 oauthlib==3.1.0           # via django-oauth-toolkit, requests-oauthlib, social-auth-core
 openshift==0.10.1
@@ -79,15 +82,16 @@ prometheus-client==0.7.1
 psutil==5.6.7             # via ansible-runner
 psycopg2==2.8.4
 ptyprocess==0.6.0         # via pexpect
-pyasn1-modules==0.2.7     # via google-auth, python-ldap
-pyasn1==0.4.8             # via pyasn1-modules, python-ldap, rsa
+pyasn1-modules==0.2.8     # via google-auth, python-ldap, service-identity
+pyasn1==0.4.8             # via pyasn1-modules, python-ldap, rsa, service-identity
 pycparser==2.19           # via cffi
 pygerduty==0.38.2
 pyhamcrest==1.9.0         # via twisted
 pyjwt==1.7.1              # via adal, social-auth-core, twilio
-pyparsing==2.4.5
+pyopenssl==19.1.0         # via twisted
+pyparsing==2.4.6
 pyrad==2.2                # via django-radius
-pyrsistent==0.15.6        # via jsonschema
+pyrsistent==0.15.7        # via jsonschema
 python-daemon==2.2.4      # via ansible-runner
 python-dateutil==2.8.1    # via adal, kubernetes
 python-ldap==3.2.0        # via django-auth-ldap
@@ -97,7 +101,8 @@ python-string-utils==0.6.0  # via openshift
 python3-openid==3.1.0     # via social-auth-core
 python3-saml==1.9.0
 pytz==2019.3              # via django, irc, tempora, twilio
-pyyaml==5.2               # via ansible-runner, djangorestframework-yaml, kubernetes
+pyyaml==5.3               # via ansible-runner, djangorestframework-yaml, kubernetes
+redis==3.3.11
 requests-futures==1.0.0
 requests-oauthlib==1.3.0  # via kubernetes, msrest, social-auth-core
 requests==2.22.0
@@ -105,6 +110,7 @@ rsa==4.0                  # via google-auth
 ruamel.yaml.clib==0.2.0   # via ruamel.yaml
 ruamel.yaml==0.16.5       # via openshift
 schedule==0.6.0
+service-identity==18.1.0  # via twisted
 six==1.13.0               # via ansible-runner, asgi-amqp, asgiref, autobahn, automat, cryptography, django-extensions, google-auth, isodate, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.text, jsonschema, kubernetes, openshift, pygerduty, pyhamcrest, pyrad, pyrsistent, python-dateutil, python-memcached, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, tempora, twilio, txaio, websocket-client
 slackclient==1.1.2
 smmap2==2.0.5             # via gitdb2
@@ -112,17 +118,19 @@ social-auth-app-django==3.1.0
 social-auth-core==3.2.0
 sqlparse==0.3.0           # via django
 tacacs_plus==1.0
-tempora==1.14.1           # via irc, jaraco.logging
-twilio==6.35.1
-twisted==19.10.0          # via daphne
+tempora==2.1.0            # via irc, jaraco.logging
+twilio==6.35.2
+twisted[tls]==19.10.0     # via daphne
 txaio==18.8.1             # via autobahn
+typing-extensions==3.7.4.1  # via aiohttp
 urllib3==1.25.7           # via kubernetes, requests
 uwsgi==2.0.18
 uwsgitop==0.11
 vine==1.3.0               # via amqp
-websocket-client==0.56.0  # via kubernetes, slackclient
+websocket-client==0.57.0  # via kubernetes, slackclient
 xmlsec==1.3.3             # via python3-saml
-zipp==0.6.0               # via importlib-metadata
+yarl==1.4.2               # via aiohttp
+zipp==1.0.0               # via importlib-metadata
 zope.interface==4.7.1     # via twisted
 
 # The following packages are considered to be unsafe in a requirements file:

tools/docker-compose-cluster.yml

@@ -7,89 +7,99 @@ services:
       dockerfile: Dockerfile-haproxy
     container_name: tools_haproxy_1
     depends_on:
-      - "awx_1"
-      - "awx_2"
-      - "awx_3"
+      - "awx-1"
+      - "awx-2"
+      - "awx-3"
     ports:
       - "8013:8013"
       - "8043:8043"
       - "1936:1936"
       - "15672:15672"
-  awx_1:
+  awx-1:
     user: ${CURRENT_UID}
     container_name: tools_awx_1_1
     privileged: true
     image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG}
-    hostname: awx_1
+    hostname: awx-1
     environment:
       CURRENT_UID:
-      RABBITMQ_HOST: rabbitmq_1
-      RABBITMQ_USER: guest
-      RABBITMQ_PASS: guest
-      RABBITMQ_VHOST: /
       SDB_HOST: 0.0.0.0
       SDB_PORT: 5899
       AWX_GROUP_QUEUES: alpha,tower
+    command: /start_development.sh
+    working_dir: "/awx_devel"
     volumes:
       - "../:/awx_devel"
+      - "./redis/redis_socket_ha_1:/var/run/redis/"
     ports:
       - "5899-5999:5899-5999"
-  awx_2:
+  awx-2:
     user: ${CURRENT_UID}
     container_name: tools_awx_2_1
     privileged: true
     image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG}
-    hostname: awx_2
+    hostname: awx-2
+    command: /start_development.sh
+    working_dir: "/awx_devel"
     environment:
       CURRENT_UID:
-      RABBITMQ_HOST: rabbitmq_2
-      RABBITMQ_USER: guest
-      RABBITMQ_PASS: guest
-      RABBITMQ_VHOST: /
       SDB_HOST: 0.0.0.0
       SDB_PORT: 7899
       AWX_GROUP_QUEUES: bravo,tower
     volumes:
       - "../:/awx_devel"
+      - "./redis/redis_socket_ha_2:/var/run/redis/"
     ports:
       - "7899-7999:7899-7999"
-  awx_3:
+  awx-3:
     user: ${CURRENT_UID}
     container_name: tools_awx_3_1
     privileged: true
     image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG}
-    hostname: awx_3
+    hostname: awx-3
+    entrypoint: ["bash"]
+    command: /start_development.sh
+    working_dir: "/awx_devel"
     environment:
       CURRENT_UID:
-      RABBITMQ_HOST: rabbitmq_3
-      RABBITMQ_USER: guest
-      RABBITMQ_PASS: guest
-      RABBITMQ_VHOST: /
       SDB_HOST: 0.0.0.0
       SDB_PORT: 8899
       AWX_GROUP_QUEUES: charlie,tower
     volumes:
       - "../:/awx_devel"
+      - "./redis/redis_socket_ha_3:/var/run/redis/"
     ports:
       - "8899-8999:8899-8999"
-  rabbitmq_1:
-    image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest
-    hostname: rabbitmq_1
-    container_name: tools_rabbitmq_1_1
-  rabbitmq_2:
-    image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest
-    hostname: rabbitmq_2
-    container_name: tools_rabbitmq_2_1
-    environment:
-      - CLUSTERED=true
-      - CLUSTER_WITH=rabbitmq_1
-  rabbitmq_3:
-    image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest
-    hostname: rabbitmq_3
-    container_name: tools_rabbitmq_3_1
-    environment:
-      - CLUSTERED=true
-      - CLUSTER_WITH=rabbitmq_1
+  redis_1:
+    user: ${CURRENT_UID}
+    image: redis:latest
+    container_name: tools_redis_1_1
+    command: ["/usr/local/etc/redis/redis.conf"]
+    volumes:
+      - "./redis/redis.conf:/usr/local/etc/redis/redis.conf"
+      - "./redis/redis_socket_ha_1:/var/run/redis/"
+    ports:
+      - "63791:63791"
+  redis_2:
+    user: ${CURRENT_UID}
+    image: redis:latest
+    container_name: tools_redis_2_1
+    command: ["/usr/local/etc/redis/redis.conf"]
+    volumes:
+      - "./redis/redis.conf:/usr/local/etc/redis/redis.conf"
+      - "./redis/redis_socket_ha_2:/var/run/redis/"
+    ports:
+      - "63792:63792"
+  redis_3:
+    user: ${CURRENT_UID}
+    image: redis:latest
+    container_name: tools_redis_3_1
+    command: ["/usr/local/etc/redis/redis.conf"]
+    volumes:
+      - "./redis/redis.conf:/usr/local/etc/redis/redis.conf"
+      - "./redis/redis_socket_ha_3:/var/run/redis/"
+    ports:
+      - "63793:63793"
   postgres:
     image: postgres:10
     container_name: tools_postgres_1

tools/docker-compose.yml

@@ -11,10 +11,6 @@ services:
     environment:
       CURRENT_UID:
       OS:
-      RABBITMQ_HOST: rabbitmq
-      RABBITMQ_USER: guest
-      RABBITMQ_PASS: guest
-      RABBITMQ_VHOST: /
       SDB_HOST: 0.0.0.0
       SDB_PORT: 7899
       AWX_GROUP_QUEUES: tower
@@ -28,7 +24,7 @@ services:
     links:
       - postgres
       - memcached
-      - rabbitmq
+      - redis
     #   - sync
     # volumes_from:
     #   - sync
@@ -36,6 +32,7 @@ services:
     volumes:
       - "../:/awx_devel"
       - "../awx/projects/:/var/lib/awx/projects/"
+      - "./redis/redis_socket_standalone:/var/run/redis/"
     privileged: true
   # A useful container that simply passes through log messages to the console
   # helpful for testing awx/tower logging
@@ -57,8 +54,13 @@ services:
     container_name: tools_memcached_1
     ports:
       - "11211:11211"
-  rabbitmq:
-    image: rabbitmq:3-management
-    container_name: tools_rabbitmq_1
+  redis:
+    image: redis:latest
+    container_name: tools_redis_1
     ports:
-      - "15672:15672"
+      - "6379:6379"
+    user: ${CURRENT_UID}
+    volumes:
+      - "./redis/redis.conf:/usr/local/etc/redis/redis.conf"
+      - "./redis/redis_socket_standalone:/var/run/redis/"
+    command: ["/usr/local/etc/redis/redis.conf"]

tools/docker-compose/bootstrap_development.sh

@@ -4,7 +4,7 @@ set +x
 # Wait for the databases to come up
 ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=postgres port=5432" all
 ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=memcached port=11211" all
-ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=${RABBITMQ_HOST} port=5672" all
+ansible -i "127.0.0.1," -c local -v -m wait_for -a "path=/var/run/redis/redis.sock" all
 
 # In case AWX in the container wants to connect to itself, use "docker exec" to attach to the container otherwise
 # TODO: FIX

tools/docker-compose/haproxy.cfg

@@ -22,11 +22,6 @@ frontend localnodes_ssl
     mode tcp
     default_backend nodes_ssl
 
-frontend rabbitctl
-    bind *:15672
-    mode http
-    default_backend rabbitctl_nodes
-
 backend nodes
     mode http
     balance roundrobin
@@ -35,28 +30,16 @@ backend nodes
     http-request set-header X-Forwarded-Port %[dst_port]
     http-request add-header X-Forwarded-Proto https if { ssl_fc }
     option httpchk HEAD / HTTP/1.1\r\nHost:localhost
-    server awx_1 awx_1:8013 check
-    server awx_2 awx_2:8013 check
-    server awx_3 awx_3:8013 check
+    server awx-1 awx-1:8013 check
+    server awx-2 awx-2:8013 check
+    server awx-3 awx-3:8013 check
 
 backend nodes_ssl
     mode tcp
     balance roundrobin
-    server awx_1 awx_1:8043
-    server awx_2 awx_2:8043
-    server awx_3 awx_3:8043
-
-backend rabbitctl_nodes
-    mode http
-    balance roundrobin
-    option forwardfor
-    option http-pretend-keepalive
-    http-request set-header X-Forwarded-Port %[dst_port]
-    http-request add-header X-Forwarded-Proto https if { ssl_fc }
-    #option httpchk HEAD / HTTP/1.1\r\nHost:localhost
-    server rabbitmq_1 rabbitmq_1:15672
-    server rabbitmq_2 rabbitmq_2:15672
-    server rabbitmq_3 rabbitmq_3:15672
+    server awx-1 awx-1:8043
+    server awx-2 awx-2:8043
+    server awx-3 awx-3:8043
 
 listen stats
     bind *:1936

tools/docker-compose/supervisor.conf

@@ -27,10 +27,14 @@ redirect_stderr=true
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
-[program:awx-runworker]
-command = make runworker
+[program:awx-wsbroadcast]
+command = make wsbroadcast
 autostart = true
 autorestart = true
+stopwaitsecs = 1
+stopsignal=KILL
+stopasgroup=true
+killasgroup=true
 redirect_stderr=true
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
@@ -52,6 +56,10 @@ command = make daphne
 autostart = true
 autorestart = true
 redirect_stderr=true
+stopwaitsecs = 1
+stopsignal=KILL
+stopasgroup=true
+killasgroup=true
 stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
@@ -64,7 +72,7 @@ stdout_logfile=/dev/fd/1
 stdout_logfile_maxbytes=0
 
 [group:tower-processes]
-programs=awx-dispatcher,awx-receiver,awx-runworker,awx-uwsgi,awx-daphne,awx-nginx
+programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast
 priority=5
 
 [unix_http_server]

tools/redis/redis.conf

@@ -0,0 +1,10 @@
+unixsocket /var/run/redis/redis.sock
+unixsocketperm 770
+port 0
+# Do not actually listen to any tcp port
+# but include the bind directive because without it redis will
+# listen on the public interface. Port 0 causes it to NOT listen on
+# the public interface. Adding the below line is an extra precaution.
+# If a developer comes by later and wants to listen on a tcp port and changes
+# the above port, it will ONLY listen on the local interface.
+bind 127.0.0.1

tools/redis/redis_socket_ha_1/.dir_placeholder

@@ -0,0 +1 @@
+This dir must pre-exist and be owned by the user you are launching awx dev env as. If the dir does not exist before launching the awx dev environment then docker will create the dir and it will be owned by root. Since we start our awx dev environment with user: ${CURRENT_UID} the redis container will be unable to create a socket file in a directory owned by root.

tools/redis/redis_socket_ha_2/.dir_placeholder

@@ -0,0 +1 @@
+This dir must pre-exist and be owned by the user you are launching awx dev env as. If the dir does not exist before launching the awx dev environment then docker will create the dir and it will be owned by root. Since we start our awx dev environment with user: ${CURRENT_UID} the redis container will be unable to create a socket file in a directory owned by root.

tools/redis/redis_socket_ha_3/.dir_placeholder

@@ -0,0 +1 @@
+This dir must pre-exist and be owned by the user you are launching awx dev env as. If the dir does not exist before launching the awx dev environment then docker will create the dir and it will be owned by root. Since we start our awx dev environment with user: ${CURRENT_UID} the redis container will be unable to create a socket file in a directory owned by root.

tools/redis/redis_socket_standalone/.dir_placeholder

@@ -0,0 +1 @@
+This dir must pre-exist and be owned by the user you are launching awx dev env as. If the dir does not exist before launching the awx dev environment then docker will create the dir and it will be owned by root. Since we start our awx dev environment with user: ${CURRENT_UID} the redis container will be unable to create a socket file in a directory owned by root.