From c8eeacacca6ae252954faedc997ec4fd1262edda Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 8 Nov 2019 10:36:39 -0500 Subject: [PATCH 01/41] POC channels 2 --- awx/asgi.py | 10 +- awx/main/channels.py | 90 ++++++ awx/main/consumers.py | 271 +++++++++++++----- awx/main/dispatch/control.py | 7 +- awx/main/dispatch/kombu.py | 42 --- awx/main/dispatch/publish.py | 10 +- .../commands/run_callback_receiver.py | 5 +- .../management/commands/run_dispatcher.py | 3 +- awx/main/queue.py | 8 +- awx/main/routing.py | 20 +- awx/main/signals.py | 10 - awx/settings/defaults.py | 30 +- awx/settings/local_settings.py.docker_compose | 23 +- requirements/requirements.in | 8 +- requirements/requirements.txt | 76 ++--- tools/docker-compose-cluster.yml | 95 +++--- tools/docker-compose.yml | 15 +- tools/docker-compose/bootstrap_development.sh | 1 - tools/docker-compose/haproxy.cfg | 29 +- tools/docker-compose/supervisor.conf | 10 +- tools/redis/redis_1.conf | 4 + tools/redis/redis_2.conf | 4 + tools/redis/redis_3.conf | 4 + 23 files changed, 497 insertions(+), 278 deletions(-) create mode 100644 awx/main/channels.py delete mode 100644 awx/main/dispatch/kombu.py create mode 100644 tools/redis/redis_1.conf create mode 100644 tools/redis/redis_2.conf create mode 100644 tools/redis/redis_3.conf diff --git a/awx/asgi.py b/awx/asgi.py index 3190a7032c..40640a4a19 100644 --- a/awx/asgi.py +++ b/awx/asgi.py @@ -2,14 +2,15 @@ # All Rights Reserved. import os import logging +import django from awx import __version__ as tower_version # Prepare the AWX environment. from awx import prepare_env, MODE prepare_env() # NOQA -from django.core.wsgi import get_wsgi_application # NOQA -from channels.asgi import get_channel_layer +from channels.routing import get_default_application + """ ASGI config for AWX project. @@ -32,6 +33,5 @@ if MODE == 'production': os.environ.setdefault("DJANGO_SETTINGS_MODULE", "awx.settings") - - -channel_layer = get_channel_layer() +django.setup() +channel_layer = get_default_application() diff --git a/awx/main/channels.py b/awx/main/channels.py new file mode 100644 index 0000000000..e76b06cf1b --- /dev/null +++ b/awx/main/channels.py @@ -0,0 +1,90 @@ + +import os +import json +import logging +import aiohttp +import asyncio + +from channels_redis.core import RedisChannelLayer +from channels.layers import get_channel_layer + +from django.utils.encoding import force_bytes +from django.conf import settings +from django.apps import apps +from django.core.serializers.json import DjangoJSONEncoder + + +logger = logging.getLogger('awx.main') + + +def wrap_broadcast_msg(group, message): + # TODO: Maybe wrap as "group","message" so that we don't need to + # encode/decode as json. + return json.dumps(dict(group=group, message=message), cls=DjangoJSONEncoder) + + +def unwrap_broadcast_msg(payload): + return (payload['group'], payload['message']) + + +def get_broadcast_hosts(): + Instance = apps.get_model('main', 'Instance') + return [h[0] for h in Instance.objects.filter(rampart_groups__controller__isnull=True) + .exclude(hostname=Instance.objects.me().hostname) + .order_by('hostname') + .values_list('hostname') + .distinct()] + + +class RedisGroupBroadcastChannelLayer(RedisChannelLayer): + def __init__(self, *args, **kwargs): + super(RedisGroupBroadcastChannelLayer, self).__init__(*args, **kwargs) + + self.broadcast_hosts = get_broadcast_hosts() + self.broadcast_websockets = set() + + loop = asyncio.get_event_loop() + for host in self.broadcast_hosts: + loop.create_task(self.connect(host, settings.BROADCAST_WEBSOCKETS_PORT)) + + async def connect(self, host, port, secret='abc123', attempt=0): + from awx.main.consumers import WebsocketSecretAuthHelper # noqa + + if attempt > 0: + await asyncio.sleep(5) + channel_layer = get_channel_layer() + uri = f"{settings.BROADCAST_WEBSOCKETS_PROTOCOL}://{host}:{port}/websocket/broadcast/" + timeout = aiohttp.ClientTimeout(total=10) + + secret_val = WebsocketSecretAuthHelper.construct_secret() + try: + async with aiohttp.ClientSession(headers={'secret': secret_val}, + timeout=timeout) as session: + async with session.ws_connect(uri, ssl=settings.BROADCAST_WEBSOCKETS_VERIFY_CERT) as websocket: + # TODO: Surface a health status of the broadcast interconnect + async for msg in websocket: + if msg.type == aiohttp.WSMsgType.ERROR: + break + elif msg.type == aiohttp.WSMsgType.TEXT: + try: + payload = json.loads(msg.data) + except json.JSONDecodeError: + logmsg = "Failed to decode broadcast message" + if logger.isEnabledFor(logging.DEBUG): + logmsg = "{} {}".format(logmsg, payload) + logger.warn(logmsg) + continue + + (group, message) = unwrap_broadcast_msg(payload) + + await channel_layer.group_send(group, {"type": "internal.message", "text": message}) + except Exception as e: + # Early on, this is our canary. I'm not sure what exceptions we can really encounter. + # Does aiohttp throws an exception if a disconnect happens? + logger.warn("Websocket broadcast client exception {}".format(e)) + finally: + # Reconnect + loop = asyncio.get_event_loop() + loop.create_task(self.connect(host, port, secret, attempt=attempt+1)) + + diff --git a/awx/main/consumers.py b/awx/main/consumers.py index a4fcdc96a6..c81085c988 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -1,97 +1,228 @@ + +import os import json import logging +import codecs +import datetime +import hmac -from channels import Group -from channels.auth import channel_session_user_from_http, channel_session_user - +from django.utils.encoding import force_bytes from django.utils.encoding import smart_str from django.http.cookie import parse_cookie from django.core.serializers.json import DjangoJSONEncoder +from django.conf import settings +from django.utils.encoding import force_bytes + +from channels.generic.websocket import AsyncJsonWebsocketConsumer +from channels.layers import get_channel_layer +from channels.db import database_sync_to_async + +from asgiref.sync import async_to_sync + +from awx.main.channels import wrap_broadcast_msg logger = logging.getLogger('awx.main.consumers') XRF_KEY = '_auth_user_xrf' +BROADCAST_GROUP = 'broadcast-group_send' -def discard_groups(message): - if 'groups' in message.channel_session: - for group in message.channel_session['groups']: - Group(group).discard(message.reply_channel) +class WebsocketSecretAuthHelper: + """ + Middlewareish for websockets to verify node websocket broadcast interconnect. + + Note: The "ish" is due to the channels routing interface. Routing occurs + _after_ authentication; making it hard to apply this auth to _only_ a subset of + websocket endpoints. + """ + + @classmethod + def construct_secret(cls): + nonce_serialized = "{}".format(int((datetime.datetime.utcnow()-datetime.datetime.fromtimestamp(0)).total_seconds())) + payload_dict = { + 'secret': settings.BROADCAST_WEBSOCKETS_SECRET, + 'nonce': nonce_serialized + } + payload_serialized = json.dumps(payload_dict) + + secret_serialized = hmac.new(force_bytes(settings.BROADCAST_WEBSOCKETS_SECRET), + msg=force_bytes(payload_serialized), + digestmod='sha256').hexdigest() + + return 'HMAC-SHA256 {}:{}'.format(nonce_serialized, secret_serialized) -@channel_session_user_from_http -def ws_connect(message): - headers = dict(message.content.get('headers', '')) - message.reply_channel.send({"accept": True}) - message.content['method'] = 'FAKE' - if message.user.is_authenticated: - message.reply_channel.send( - {"text": json.dumps({"accept": True, "user": message.user.id})} - ) - # store the valid CSRF token from the cookie so we can compare it later - # on ws_receive - cookie_token = parse_cookie( - smart_str(headers.get(b'cookie')) - ).get('csrftoken') - if cookie_token: - message.channel_session[XRF_KEY] = cookie_token - else: - logger.error("Request user is not authenticated to use websocket.") - message.reply_channel.send({"close": True}) - return None + @classmethod + def verify_secret(cls, s, nonce_tolerance=300): + hex_decoder = codecs.getdecoder("hex_codec") + + try: + (prefix, payload) = s.split(' ') + if prefix != 'HMAC-SHA256': + raise ValueError('Unsupported encryption algorithm') + (nonce_parsed, secret_parsed) = payload.split(':') + except Exception: + raise ValueError("Failed to parse secret") + + try: + payload_expected = { + 'secret': settings.BROADCAST_WEBSOCKETS_SECRET, + 'nonce': nonce_parsed, + } + payload_serialized = json.dumps(payload_expected) + except Exception: + raise ValueError("Failed to create hash to compare to secret.") + + secret_serialized = hmac.new(force_bytes(settings.BROADCAST_WEBSOCKETS_SECRET), + msg=force_bytes(payload_serialized), + digestmod='sha256').hexdigest() + + if secret_serialized != secret_parsed: + raise ValueError("Invalid secret") + + # Avoid timing attack and check the nonce after all the heavy lifting + now = datetime.datetime.utcnow() + nonce_parsed = datetime.datetime.fromtimestamp(int(nonce_parsed)) + if (now-nonce_parsed).total_seconds() > nonce_tolerance: + raise ValueError("Potential replay attack or machine(s) time out of sync.") + + return True + + @classmethod + def is_authorized(cls, scope): + secret = '' + for k, v in scope['headers']: + if k.decode("utf-8") == 'secret': + secret = v.decode("utf-8") + break + WebsocketSecretAuthHelper.verify_secret(secret) -@channel_session_user -def ws_disconnect(message): - discard_groups(message) +class BroadcastConsumer(AsyncJsonWebsocketConsumer): + + async def connect(self): + try: + WebsocketSecretAuthHelper.is_authorized(self.scope) + except Exception: + await self.close() + return + + # TODO: log ip of connected client + logger.info("Client connected") + await self.accept() + await self.channel_layer.group_add(BROADCAST_GROUP, self.channel_name) + + async def disconnect(self, code): + # TODO: log ip of disconnected client + logger.info("Client disconnected") + + async def internal_message(self, event): + await self.send(event['text']) -@channel_session_user -def ws_receive(message): - from awx.main.access import consumer_access - user = message.user - raw_data = message.content['text'] - data = json.loads(raw_data) +class EventConsumer(AsyncJsonWebsocketConsumer): + async def connect(self): + user = self.scope['user'] + if user and not user.is_anonymous: + await self.accept() + await self.send_json({"accept": True, "user": user.id}) + # store the valid CSRF token from the cookie so we can compare it later + # on ws_receive + cookie_token = self.scope['cookies'].get('csrftoken') + if cookie_token: + self.scope['session'][XRF_KEY] = cookie_token + else: + logger.error("Request user is not authenticated to use websocket.") + # TODO: Carry over from channels 1 implementation + # We should never .accept() the client and close without sending a close message + await self.accept() + await self.send_json({"close": True}) + await self.close() - xrftoken = data.get('xrftoken') - if ( - not xrftoken or - XRF_KEY not in message.channel_session or - xrftoken != message.channel_session[XRF_KEY] - ): - logger.error( + @database_sync_to_async + def user_can_see_object_id(self, user_access): + return user_access.get_queryset().filter(pk=oid).exists() + + async def receive_json(self, data): + from awx.main.access import consumer_access + user = self.scope['user'] + xrftoken = data.get('xrftoken') + if ( + not xrftoken or + XRF_KEY not in self.scope["session"] or + xrftoken != self.scope["session"][XRF_KEY] + ): + logger.error( "access denied to channel, XRF mismatch for {}".format(user.username) - ) - message.reply_channel.send({ - "text": json.dumps({"error": "access denied to channel"}) - }) - return + ) + await self.send_json({"error": "access denied to channel"}) + return - if 'groups' in data: - discard_groups(message) - groups = data['groups'] - current_groups = set(message.channel_session.pop('groups') if 'groups' in message.channel_session else []) - for group_name,v in groups.items(): - if type(v) is list: - for oid in v: - name = '{}-{}'.format(group_name, oid) - access_cls = consumer_access(group_name) - if access_cls is not None: - user_access = access_cls(user) - if not user_access.get_queryset().filter(pk=oid).exists(): - message.reply_channel.send({"text": json.dumps( - {"error": "access denied to channel {0} for resource id {1}".format(group_name, oid)})}) - continue - current_groups.add(name) - Group(name).add(message.reply_channel) - else: - current_groups.add(group_name) - Group(group_name).add(message.reply_channel) - message.channel_session['groups'] = list(current_groups) + if 'groups' in data: + groups = data['groups'] + new_groups = set() + current_groups = set(self.scope['session'].pop('groups') if 'groups' in self.scope['session'] else []) + for group_name,v in groups.items(): + if type(v) is list: + for oid in v: + name = '{}-{}'.format(group_name, oid) + access_cls = consumer_access(group_name) + if access_cls is not None: + user_access = access_cls(user) + if not self.user_can_see_object_id(user_access): + await self.send_json({"error": "access denied to channel {0} for resource id {1}".format(group_name, oid)}) + continue + + new_groups.add(name) + else: + if group_name == BROADCAST_GROUP: + logger.warn("Non-priveleged client asked to join broadcast group!") + return + + new_groups.add(name) + + old_groups = current_groups - new_groups + for group_name in old_groups: + await self.channel_layer.group_discard( + group_name, + self.channel_name, + ) + + new_groups_exclusive = new_groups - current_groups + for group_name in new_groups_exclusive: + await self.channel_layer.group_add( + group_name, + self.channel_name + ) + logger.debug(f"Channel {self.channel_name} left groups {old_groups} and joined {new_groups_exclusive}") + self.scope['session']['groups'] = new_groups + + async def internal_message(self, event): + await self.send(event['text']) def emit_channel_notification(group, payload): try: - Group(group).send({"text": json.dumps(payload, cls=DjangoJSONEncoder)}) + payload = json.dumps(payload, cls=DjangoJSONEncoder) except ValueError: logger.error("Invalid payload emitting channel {} on topic: {}".format(group, payload)) + return + + channel_layer = get_channel_layer() + + async_to_sync(channel_layer.group_send)( + group, + { + "type": "internal.message", + "text": payload + }, + ) + + async_to_sync(channel_layer.group_send)( + BROADCAST_GROUP, + { + "type": "internal.message", + "text": wrap_broadcast_msg(group, payload), + }, + ) + diff --git a/awx/main/dispatch/control.py b/awx/main/dispatch/control.py index 5f081e84f2..f938aab6b5 100644 --- a/awx/main/dispatch/control.py +++ b/awx/main/dispatch/control.py @@ -4,8 +4,7 @@ import socket from django.conf import settings from awx.main.dispatch import get_local_queuename -from awx.main.dispatch.kombu import Connection -from kombu import Queue, Exchange, Producer, Consumer +from kombu import Queue, Exchange, Producer, Consumer, Connection logger = logging.getLogger('awx.main.dispatch') @@ -40,7 +39,7 @@ class Control(object): logger.warn('checking {} {} for {}'.format(self.service, command, self.queuename)) reply_queue = Queue(name="amq.rabbitmq.reply-to") self.result = None - with Connection(settings.BROKER_URL) as conn: + with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: with Consumer(conn, reply_queue, callbacks=[self.process_message], no_ack=True): self.publish({'control': command}, conn, reply_to='amq.rabbitmq.reply-to') try: @@ -51,7 +50,7 @@ class Control(object): return self.result def control(self, msg, **kwargs): - with Connection(settings.BROKER_URL) as conn: + with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: self.publish(msg, conn) def process_message(self, body, message): diff --git a/awx/main/dispatch/kombu.py b/awx/main/dispatch/kombu.py deleted file mode 100644 index 94fc7a035e..0000000000 --- a/awx/main/dispatch/kombu.py +++ /dev/null @@ -1,42 +0,0 @@ -from amqp.exceptions import PreconditionFailed -from django.conf import settings -from kombu.connection import Connection as KombuConnection -from kombu.transport import pyamqp - -import logging - -logger = logging.getLogger('awx.main.dispatch') - - -__all__ = ['Connection'] - - -class Connection(KombuConnection): - - def __init__(self, *args, **kwargs): - super(Connection, self).__init__(*args, **kwargs) - class _Channel(pyamqp.Channel): - - def queue_declare(self, queue, *args, **kwargs): - kwargs['durable'] = settings.BROKER_DURABILITY - try: - return super(_Channel, self).queue_declare(queue, *args, **kwargs) - except PreconditionFailed as e: - if "inequivalent arg 'durable'" in getattr(e, 'reply_text', None): - logger.error( - 'queue {} durability is not {}, deleting and recreating'.format( - - queue, - kwargs['durable'] - ) - ) - self.queue_delete(queue) - return super(_Channel, self).queue_declare(queue, *args, **kwargs) - - class _Connection(pyamqp.Connection): - Channel = _Channel - - class _Transport(pyamqp.Transport): - Connection = _Connection - - self.transport_cls = _Transport diff --git a/awx/main/dispatch/publish.py b/awx/main/dispatch/publish.py index 9bbd7ae45f..be64594ee3 100644 --- a/awx/main/dispatch/publish.py +++ b/awx/main/dispatch/publish.py @@ -4,9 +4,8 @@ import sys from uuid import uuid4 from django.conf import settings -from kombu import Exchange, Producer +from kombu import Exchange, Producer, Connection, Queue, Consumer -from awx.main.dispatch.kombu import Connection logger = logging.getLogger('awx.main.dispatch') @@ -86,8 +85,13 @@ class task: if callable(queue): queue = queue() if not settings.IS_TESTING(sys.argv): - with Connection(settings.BROKER_URL) as conn: + with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: exchange = Exchange(queue, type=exchange_type or 'direct') + + # HACK: With Redis as the broker declaring an exchange isn't enough to create the queue + # Creating a Consumer _will_ create a queue so that publish will succeed. Note that we + # don't call consume() on the consumer so we don't actually eat any messages + Consumer(conn, queues=[Queue(queue, exchange, routing_key=queue)], accept=['json']) producer = Producer(conn) logger.debug('publish {}({}, queue={})'.format( cls.name, diff --git a/awx/main/management/commands/run_callback_receiver.py b/awx/main/management/commands/run_callback_receiver.py index 51608a8b7a..58e311f2bb 100644 --- a/awx/main/management/commands/run_callback_receiver.py +++ b/awx/main/management/commands/run_callback_receiver.py @@ -3,9 +3,8 @@ from django.conf import settings from django.core.management.base import BaseCommand -from kombu import Exchange, Queue +from kombu import Exchange, Queue, Connection -from awx.main.dispatch.kombu import Connection from awx.main.dispatch.worker import AWXConsumer, CallbackBrokerWorker @@ -18,7 +17,7 @@ class Command(BaseCommand): help = 'Launch the job callback receiver' def handle(self, *arg, **options): - with Connection(settings.BROKER_URL) as conn: + with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: consumer = None try: consumer = AWXConsumer( diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py index 9fd9c3256d..7e69897687 100644 --- a/awx/main/management/commands/run_dispatcher.py +++ b/awx/main/management/commands/run_dispatcher.py @@ -11,7 +11,6 @@ from kombu import Exchange, Queue from awx.main.utils.handlers import AWXProxyHandler from awx.main.dispatch import get_local_queuename, reaper from awx.main.dispatch.control import Control -from awx.main.dispatch.kombu import Connection from awx.main.dispatch.pool import AutoscalePool from awx.main.dispatch.worker import AWXConsumer, TaskWorker from awx.main.dispatch import periodic @@ -63,7 +62,7 @@ class Command(BaseCommand): # in cpython itself: # https://bugs.python.org/issue37429 AWXProxyHandler.disable() - with Connection(settings.BROKER_URL) as conn: + with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: try: bcast = 'tower_broadcast_all' queues = [ diff --git a/awx/main/queue.py b/awx/main/queue.py index 0da0e22e48..3d8a8384eb 100644 --- a/awx/main/queue.py +++ b/awx/main/queue.py @@ -10,8 +10,7 @@ import os from django.conf import settings # Kombu -from awx.main.dispatch.kombu import Connection -from kombu import Exchange, Producer +from kombu import Exchange, Producer, Connection from kombu.serialization import registry __all__ = ['CallbackQueueDispatcher'] @@ -41,6 +40,7 @@ class CallbackQueueDispatcher(object): def __init__(self): self.callback_connection = getattr(settings, 'BROKER_URL', None) + self.callback_connection_options = getattr(settings, 'BROKER_TRANSPORT_OPTIONS', {}) self.connection_queue = getattr(settings, 'CALLBACK_QUEUE', '') self.connection = None self.exchange = None @@ -57,7 +57,7 @@ class CallbackQueueDispatcher(object): if self.connection_pid != active_pid: self.connection = None if self.connection is None: - self.connection = Connection(self.callback_connection) + self.connection = Connection(self.callback_connection, transport_options=self.callback_connection_options) self.exchange = Exchange(self.connection_queue, type='direct') producer = Producer(self.connection) @@ -66,7 +66,7 @@ class CallbackQueueDispatcher(object): compression='bzip2', exchange=self.exchange, declare=[self.exchange], - delivery_mode="persistent" if settings.PERSISTENT_CALLBACK_MESSAGES else "transient", + delivery_mode="transient", routing_key=self.connection_queue) return except Exception as e: diff --git a/awx/main/routing.py b/awx/main/routing.py index 0a49f25c6c..1efb6159d3 100644 --- a/awx/main/routing.py +++ b/awx/main/routing.py @@ -1,8 +1,16 @@ -from channels.routing import route +from django.urls import re_path +from django.conf.urls import url +from channels.auth import AuthMiddlewareStack +from channels.routing import ProtocolTypeRouter, URLRouter +from . import consumers - -channel_routing = [ - route("websocket.connect", "awx.main.consumers.ws_connect", path=r'^/websocket/$'), - route("websocket.disconnect", "awx.main.consumers.ws_disconnect", path=r'^/websocket/$'), - route("websocket.receive", "awx.main.consumers.ws_receive", path=r'^/websocket/$'), +websocket_urlpatterns = [ + url(r'websocket/$', consumers.EventConsumer), + url(r'websocket/broadcast/$', consumers.BroadcastConsumer), ] + +application = ProtocolTypeRouter({ + 'websocket': AuthMiddlewareStack( + URLRouter(websocket_urlpatterns) + ), +}) diff --git a/awx/main/signals.py b/awx/main/signals.py index 64a35c1f1d..1983b335f1 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -593,16 +593,6 @@ def deny_orphaned_approvals(sender, instance, **kwargs): @receiver(post_save, sender=Session) def save_user_session_membership(sender, **kwargs): session = kwargs.get('instance', None) - if pkg_resources.get_distribution('channels').version >= '2': - # If you get into this code block, it means we upgraded channels, but - # didn't make the settings.SESSIONS_PER_USER feature work - raise RuntimeError( - 'save_user_session_membership must be updated for channels>=2: ' - 'http://channels.readthedocs.io/en/latest/one-to-two.html#requirements' - ) - if 'runworker' in sys.argv: - # don't track user session membership for websocket per-channel sessions - return if not session: return user_id = session.get_decoded().get(SESSION_KEY, None) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index c5c0ad5e89..252e39d708 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -421,7 +421,8 @@ os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199') BROKER_DURABILITY = True BROKER_POOL_LIMIT = None -BROKER_URL = 'amqp://guest:guest@localhost:5672//' +BROKER_URL = 'redis://localhost:6379;' +BROKER_TRANSPORT_OPTIONS = {} CELERY_DEFAULT_QUEUE = 'awx_private_queue' CELERYBEAT_SCHEDULE = { 'tower_scheduler': { @@ -929,8 +930,6 @@ ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC = False # Internal API URL for use by inventory scripts and callback plugin. INTERNAL_API_URL = 'http://127.0.0.1:%s' % DEVSERVER_DEFAULT_PORT -PERSISTENT_CALLBACK_MESSAGES = True -USE_CALLBACK_QUEUE = True CALLBACK_QUEUE = "callback_tasks" SCHEDULER_QUEUE = "scheduler" @@ -965,6 +964,17 @@ LOG_AGGREGATOR_LEVEL = 'INFO' # raising this value can help CHANNEL_LAYER_RECEIVE_MAX_RETRY = 10 +ASGI_APPLICATION = "awx.main.routing.application" + +CHANNEL_LAYERS = { + "default": { + "BACKEND": "awx.main.channels.RedisGroupBroadcastChannelLayer", + "CONFIG": { + "hosts": [("localhost", 6379)], + }, + }, +} + # Logging configuration. LOGGING = { 'version': 1, @@ -1239,3 +1249,17 @@ MIDDLEWARE = [ 'awx.main.middleware.URLModificationMiddleware', 'awx.main.middleware.SessionTimeoutMiddleware', ] + +# Secret header value to exchange for websockets responsible for distributing websocket messages. +# This needs to be kept secret and randomly generated +BROADCAST_WEBSOCKETS_SECRET = '' + +# Port for broadcast websockets to connect to +# Note: that the clients will follow redirect responses +BROADCAST_WEBSOCKETS_PORT = 443 + +# Whether or not broadcast websockets should check nginx certs when interconnecting +BROADCAST_WEBSOCKETS_VERIFY_CERT = False + +# Connect to other AWX nodes using http or https +BROADCAST_WEBSOCKETS_PROTOCOL = 'https' diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index 42e5a3cd74..8ebaa8747e 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -49,16 +49,18 @@ if "pytest" in sys.modules: } } -# AMQP configuration. -BROKER_URL = "amqp://{}:{}@{}/{}".format(os.environ.get("RABBITMQ_USER"), - os.environ.get("RABBITMQ_PASS"), - os.environ.get("RABBITMQ_HOST"), - urllib.parse.quote(os.environ.get("RABBITMQ_VHOST", "/"), safe='')) +# Use Redis as the message bus for now +# Default to "just works" for single tower docker +BROKER_URL = os.environ.get('BROKER_URL', "redis://redis_1:6379") CHANNEL_LAYERS = { - 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer', - 'ROUTING': 'awx.main.routing.channel_routing', - 'CONFIG': {'url': BROKER_URL}} + "default": { + "BACKEND": "awx.main.channels.RedisGroupBroadcastChannelLayer", + "CONFIG": { + "hosts": [(os.environ.get('REDIS_HOST', 'redis_1'), + int(os.environ.get('REDIS_PORT', 6379)))], + }, + }, } # Absolute filesystem path to the directory to host projects (with playbooks). @@ -238,3 +240,8 @@ TEST_OPENSTACK_PROJECT = '' # Azure credentials. TEST_AZURE_USERNAME = '' TEST_AZURE_KEY_DATA = '' + +BROADCAST_WEBSOCKETS_SECRET = '🤖starscream🤖' +BROADCAST_WEBSOCKETS_PORT = 8013 +BROADCAST_WEBSOCKETS_VERIFY_CERT = False +BROADCAST_WEBSOCKETS_PROTOCOL = 'http' diff --git a/requirements/requirements.in b/requirements/requirements.in index 09f08975a2..b6ba519768 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,10 +1,11 @@ +aiohttp ansible-runner ansiconv==1.0.0 # UPGRADE BLOCKER: from 2013, consider replacing instead of upgrading -asgi-amqp>=1.1.4 # see library notes, related to channels 2 azure-keyvault==1.1.0 # see UPGRADE BLOCKERs boto # replacement candidate https://github.com/ansible/awx/issues/2115 -channels==1.1.8 # UPGRADE BLOCKER: Last before backwards-incompatible channels 2 upgrade -daphne==1.4.2 # UPGRADE BLOCKER: last before channels 2 but not pinned by other deps +channels +channels-redis +daphne django==2.2.10 # see UPGRADE BLOCKERs django-auth-ldap django-cors-headers @@ -37,6 +38,7 @@ python3-saml schedule==0.6.0 social-auth-core==3.2.0 # see UPGRADE BLOCKERs social-auth-app-django==3.1.0 # see UPGRADE BLOCKERs +redis requests requests-futures # see library notes slackclient==1.1.2 # see UPGRADE BLOCKERs diff --git a/requirements/requirements.txt b/requirements/requirements.txt index a636429641..d778a73922 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,33 +1,36 @@ adal==1.2.2 # via msrestazure +aiohttp==3.6.2 +aioredis==1.3.1 # via channels-redis amqp==2.5.2 # via kombu ansible-runner==1.4.4 ansiconv==1.0.0 -asgi-amqp==1.1.4 -asgiref==1.1.2 # via asgi-amqp, channels, daphne -attrs==19.3.0 # via automat, jsonschema, twisted -autobahn==19.11.1 # via daphne +asgiref==3.2.3 # via channels, channels-redis, daphne +async-timeout==3.0.1 # via aiohttp, aioredis +attrs==19.3.0 # via aiohttp, automat, jsonschema, service-identity, twisted +autobahn==20.1.1 # via daphne automat==0.8.0 # via twisted -azure-common==1.1.23 # via azure-keyvault +azure-common==1.1.24 # via azure-keyvault azure-keyvault==1.1.0 azure-nspkg==3.0.2 # via azure-keyvault boto==2.49.0 -cachetools==3.1.1 # via google-auth +cachetools==4.0.0 # via google-auth certifi==2019.11.28 # via kubernetes, msrest, requests cffi==1.13.2 # via cryptography -channels==1.1.8 -chardet==3.0.4 # via requests +channels-redis==2.4.1 +channels==2.4.0 +chardet==3.0.4 # via aiohttp, requests constantly==15.1.0 # via twisted -cryptography==2.8 # via adal, autobahn, azure-keyvault -daphne==1.4.2 +cryptography==2.8 # via adal, autobahn, azure-keyvault, pyopenssl, service-identity +daphne==2.4.1 defusedxml==0.6.0 # via python3-openid, python3-saml, social-auth-core dictdiffer==0.8.1 # via openshift django-auth-ldap==2.1.0 -django-cors-headers==3.2.0 +django-cors-headers==3.2.1 django-crum==0.7.5 django-extensions==2.2.5 django-jsonfield==1.2.0 django-oauth-toolkit==1.1.3 -django-pglocks==1.0.3 +django-pglocks==1.0.4 django-polymorphic==2.1.2 django-qsstats-magic==1.1.0 django-radius==1.3.3 @@ -41,24 +44,23 @@ docutils==0.15.2 # via python-daemon future==0.16.0 # via django-radius gitdb2==2.0.6 # via gitpython gitpython==3.0.5 -google-auth==1.9.0 # via kubernetes +google-auth==1.10.0 # via kubernetes +hiredis==1.0.1 # via aioredis hyperlink==19.0.0 # via twisted -idna==2.8 # via hyperlink, requests -importlib-metadata==1.3.0 # via inflect, irc, jsonschema, kombu +idna-ssl==1.1.0 # via aiohttp +idna==2.8 # via hyperlink, idna-ssl, requests, twisted, yarl +importlib-metadata==1.4.0 # via irc, jsonschema, kombu importlib-resources==1.0.2 # via jaraco.text incremental==17.5.0 # via twisted -inflect==3.0.2 # via jaraco.itertools -irc==17.1 +irc==18.0.0 isodate==0.6.0 # via msrest, python3-saml -jaraco.classes==2.0 # via jaraco.collections -jaraco.collections==2.1 # via irc -jaraco.functools==2.0 # via irc, jaraco.text, tempora -jaraco.itertools==4.4.2 # via irc -jaraco.logging==2.0 # via irc +jaraco.classes==3.1.0 # via jaraco.collections +jaraco.collections==3.0.0 # via irc +jaraco.functools==3.0.0 # via irc, jaraco.text, tempora +jaraco.logging==3.0.0 # via irc jaraco.stream==3.0.0 # via irc jaraco.text==3.2.0 # via irc, jaraco.collections jinja2==2.10.3 -jsonpickle==1.2 # via asgi-amqp jsonschema==3.2.0 kombu==4.6.7 # via asgi-amqp kubernetes==10.0.1 # via openshift @@ -66,10 +68,11 @@ lockfile==0.12.2 # via python-daemon lxml==4.4.2 # via xmlsec markdown==3.1.1 markupsafe==1.1.1 # via jinja2 -more-itertools==8.0.2 # via irc, jaraco.functools, jaraco.itertools, zipp -msgpack-python==0.5.6 # via asgi-amqp +more-itertools==8.1.0 # via irc, jaraco.classes, jaraco.functools, zipp +msgpack==0.6.2 # via channels-redis msrest==0.6.10 # via azure-keyvault, msrestazure msrestazure==0.6.2 # via azure-keyvault +multidict==4.7.4 # via aiohttp, yarl netaddr==0.7.19 # via pyrad oauthlib==3.1.0 # via django-oauth-toolkit, requests-oauthlib, social-auth-core openshift==0.10.1 @@ -79,15 +82,16 @@ prometheus-client==0.7.1 psutil==5.6.7 # via ansible-runner psycopg2==2.8.4 ptyprocess==0.6.0 # via pexpect -pyasn1-modules==0.2.7 # via google-auth, python-ldap -pyasn1==0.4.8 # via pyasn1-modules, python-ldap, rsa +pyasn1-modules==0.2.8 # via google-auth, python-ldap, service-identity +pyasn1==0.4.8 # via pyasn1-modules, python-ldap, rsa, service-identity pycparser==2.19 # via cffi pygerduty==0.38.2 pyhamcrest==1.9.0 # via twisted pyjwt==1.7.1 # via adal, social-auth-core, twilio -pyparsing==2.4.5 +pyopenssl==19.1.0 # via twisted +pyparsing==2.4.6 pyrad==2.2 # via django-radius -pyrsistent==0.15.6 # via jsonschema +pyrsistent==0.15.7 # via jsonschema python-daemon==2.2.4 # via ansible-runner python-dateutil==2.8.1 # via adal, kubernetes python-ldap==3.2.0 # via django-auth-ldap @@ -98,6 +102,8 @@ python3-openid==3.1.0 # via social-auth-core python3-saml==1.9.0 pytz==2019.3 # via django, irc, tempora, twilio pyyaml==5.2 # via ansible-runner, djangorestframework-yaml, kubernetes +pyyaml==5.3 # via ansible-runner, djangorestframework-yaml, kubernetes +redis==3.3.11 requests-futures==1.0.0 requests-oauthlib==1.3.0 # via kubernetes, msrest, social-auth-core requests==2.22.0 @@ -112,17 +118,19 @@ social-auth-app-django==3.1.0 social-auth-core==3.2.0 sqlparse==0.3.0 # via django tacacs_plus==1.0 -tempora==1.14.1 # via irc, jaraco.logging -twilio==6.35.1 -twisted==19.10.0 # via daphne +tempora==2.1.0 # via irc, jaraco.logging +twilio==6.35.2 +twisted[tls]==19.10.0 # via daphne txaio==18.8.1 # via autobahn +typing-extensions==3.7.4.1 # via aiohttp urllib3==1.25.7 # via kubernetes, requests uwsgi==2.0.18 uwsgitop==0.11 vine==1.3.0 # via amqp -websocket-client==0.56.0 # via kubernetes, slackclient +websocket-client==0.57.0 # via kubernetes, slackclient xmlsec==1.3.3 # via python3-saml -zipp==0.6.0 # via importlib-metadata +yarl==1.4.2 # via aiohttp +zipp==1.0.0 # via importlib-metadata zope.interface==4.7.1 # via twisted # The following packages are considered to be unsafe in a requirements file: diff --git a/tools/docker-compose-cluster.yml b/tools/docker-compose-cluster.yml index 860a7f481d..caa2a7540f 100644 --- a/tools/docker-compose-cluster.yml +++ b/tools/docker-compose-cluster.yml @@ -7,45 +7,50 @@ services: dockerfile: Dockerfile-haproxy container_name: tools_haproxy_1 depends_on: - - "awx_1" - - "awx_2" - - "awx_3" + - "awx-1" + - "awx-2" + - "awx-3" ports: - "8013:8013" - "8043:8043" - "1936:1936" - "15672:15672" - awx_1: + awx-1: user: ${CURRENT_UID} container_name: tools_awx_1_1 privileged: true image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG} - hostname: awx_1 + hostname: awx-1 + #entrypoint: ["bash"] environment: CURRENT_UID: - RABBITMQ_HOST: rabbitmq_1 - RABBITMQ_USER: guest - RABBITMQ_PASS: guest - RABBITMQ_VHOST: / + # BROKER_URL will go away when we use postgres as our message broker + BROKER_URL: "redis://redis_1:63791" + REDIS_HOST: redis_1 + REDIS_PORT: 63791 SDB_HOST: 0.0.0.0 SDB_PORT: 5899 AWX_GROUP_QUEUES: alpha,tower + command: /start_development.sh + working_dir: "/awx_devel" volumes: - "../:/awx_devel" ports: - "5899-5999:5899-5999" - awx_2: + awx-2: user: ${CURRENT_UID} container_name: tools_awx_2_1 privileged: true image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG} - hostname: awx_2 + hostname: awx-2 + command: /start_development.sh + working_dir: "/awx_devel" environment: CURRENT_UID: - RABBITMQ_HOST: rabbitmq_2 - RABBITMQ_USER: guest - RABBITMQ_PASS: guest - RABBITMQ_VHOST: / + # BROKER_URL will go away when we use postgres as our message broker + BROKER_URL: "redis://redis_1:63791" + REDIS_HOST: redis_2 + REDIS_PORT: 63792 SDB_HOST: 0.0.0.0 SDB_PORT: 7899 AWX_GROUP_QUEUES: bravo,tower @@ -53,18 +58,21 @@ services: - "../:/awx_devel" ports: - "7899-7999:7899-7999" - awx_3: + awx-3: user: ${CURRENT_UID} container_name: tools_awx_3_1 privileged: true image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG} - hostname: awx_3 + hostname: awx-3 + entrypoint: ["bash"] + command: /start_development.sh + working_dir: "/awx_devel" environment: CURRENT_UID: - RABBITMQ_HOST: rabbitmq_3 - RABBITMQ_USER: guest - RABBITMQ_PASS: guest - RABBITMQ_VHOST: / + # BROKER_URL will go away when we use postgres as our message broker + BROKER_URL: "redis://redis_1:63791" + REDIS_HOST: redis_3 + REDIS_PORT: 63793 SDB_HOST: 0.0.0.0 SDB_PORT: 8899 AWX_GROUP_QUEUES: charlie,tower @@ -72,24 +80,33 @@ services: - "../:/awx_devel" ports: - "8899-8999:8899-8999" - rabbitmq_1: - image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest - hostname: rabbitmq_1 - container_name: tools_rabbitmq_1_1 - rabbitmq_2: - image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest - hostname: rabbitmq_2 - container_name: tools_rabbitmq_2_1 - environment: - - CLUSTERED=true - - CLUSTER_WITH=rabbitmq_1 - rabbitmq_3: - image: ${DEV_DOCKER_TAG_BASE}/rabbit_cluster_node:latest - hostname: rabbitmq_3 - container_name: tools_rabbitmq_3_1 - environment: - - CLUSTERED=true - - CLUSTER_WITH=rabbitmq_1 + redis_1: + image: redis:latest + hostname: redis_1 + container_name: tools_redis_1_1 + command: "redis-server /usr/local/etc/redis/redis.conf" + volumes: + - "./redis/redis_1.conf:/usr/local/etc/redis/redis.conf" + ports: + - "63791:63791" + redis_2: + image: redis:latest + hostname: redis_2 + container_name: tools_redis_2_1 + command: "redis-server /usr/local/etc/redis/redis.conf" + volumes: + - "./redis/redis_2.conf:/usr/local/etc/redis/redis.conf" + ports: + - "63792:63792" + redis_3: + image: redis:latest + hostname: redis_3 + container_name: tools_redis_3_1 + command: "redis-server /usr/local/etc/redis/redis.conf" + volumes: + - "./redis/redis_3.conf:/usr/local/etc/redis/redis.conf" + ports: + - "63793:63793" postgres: image: postgres:10 container_name: tools_postgres_1 diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index aa35901257..353510ce84 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -11,10 +11,6 @@ services: environment: CURRENT_UID: OS: - RABBITMQ_HOST: rabbitmq - RABBITMQ_USER: guest - RABBITMQ_PASS: guest - RABBITMQ_VHOST: / SDB_HOST: 0.0.0.0 SDB_PORT: 7899 AWX_GROUP_QUEUES: tower @@ -28,7 +24,7 @@ services: links: - postgres - memcached - - rabbitmq + - redis # - sync # volumes_from: # - sync @@ -57,8 +53,9 @@ services: container_name: tools_memcached_1 ports: - "11211:11211" - rabbitmq: - image: rabbitmq:3-management - container_name: tools_rabbitmq_1 + redis: + image: redis:latest + container_name: tools_redis_1 ports: - - "15672:15672" + - "6379:6379" + diff --git a/tools/docker-compose/bootstrap_development.sh b/tools/docker-compose/bootstrap_development.sh index dfcbe11420..cd0ce68e8b 100755 --- a/tools/docker-compose/bootstrap_development.sh +++ b/tools/docker-compose/bootstrap_development.sh @@ -4,7 +4,6 @@ set +x # Wait for the databases to come up ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=postgres port=5432" all ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=memcached port=11211" all -ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=${RABBITMQ_HOST} port=5672" all # In case AWX in the container wants to connect to itself, use "docker exec" to attach to the container otherwise # TODO: FIX diff --git a/tools/docker-compose/haproxy.cfg b/tools/docker-compose/haproxy.cfg index 9f4fcfa6e3..d37cbf691a 100644 --- a/tools/docker-compose/haproxy.cfg +++ b/tools/docker-compose/haproxy.cfg @@ -22,11 +22,6 @@ frontend localnodes_ssl mode tcp default_backend nodes_ssl -frontend rabbitctl - bind *:15672 - mode http - default_backend rabbitctl_nodes - backend nodes mode http balance roundrobin @@ -35,28 +30,16 @@ backend nodes http-request set-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto https if { ssl_fc } option httpchk HEAD / HTTP/1.1\r\nHost:localhost - server awx_1 awx_1:8013 check - server awx_2 awx_2:8013 check - server awx_3 awx_3:8013 check + server awx-1 awx-1:8013 check + server awx-2 awx-2:8013 check + server awx-3 awx-3:8013 check backend nodes_ssl mode tcp balance roundrobin - server awx_1 awx_1:8043 - server awx_2 awx_2:8043 - server awx_3 awx_3:8043 - -backend rabbitctl_nodes - mode http - balance roundrobin - option forwardfor - option http-pretend-keepalive - http-request set-header X-Forwarded-Port %[dst_port] - http-request add-header X-Forwarded-Proto https if { ssl_fc } - #option httpchk HEAD / HTTP/1.1\r\nHost:localhost - server rabbitmq_1 rabbitmq_1:15672 - server rabbitmq_2 rabbitmq_2:15672 - server rabbitmq_3 rabbitmq_3:15672 + server awx-1 awx-1:8043 + server awx-2 awx-2:8043 + server awx-3 awx-3:8043 listen stats bind *:1936 diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf index 84ebaaf291..c0e649950d 100644 --- a/tools/docker-compose/supervisor.conf +++ b/tools/docker-compose/supervisor.conf @@ -27,14 +27,6 @@ redirect_stderr=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 -[program:awx-runworker] -command = make runworker -autostart = true -autorestart = true -redirect_stderr=true -stdout_logfile=/dev/fd/1 -stdout_logfile_maxbytes=0 - [program:awx-uwsgi] command = make uwsgi autostart = true @@ -64,7 +56,7 @@ stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 [group:tower-processes] -programs=awx-dispatcher,awx-receiver,awx-runworker,awx-uwsgi,awx-daphne,awx-nginx +programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx priority=5 [unix_http_server] diff --git a/tools/redis/redis_1.conf b/tools/redis/redis_1.conf new file mode 100644 index 0000000000..8b6f3784d7 --- /dev/null +++ b/tools/redis/redis_1.conf @@ -0,0 +1,4 @@ +protected-mode no +port 63791 +dir . +logfile "/tmp/redis.log" diff --git a/tools/redis/redis_2.conf b/tools/redis/redis_2.conf new file mode 100644 index 0000000000..7f02a91f4b --- /dev/null +++ b/tools/redis/redis_2.conf @@ -0,0 +1,4 @@ +protected-mode no +port 63792 +dir . +logfile "/tmp/redis.log" diff --git a/tools/redis/redis_3.conf b/tools/redis/redis_3.conf new file mode 100644 index 0000000000..203d723421 --- /dev/null +++ b/tools/redis/redis_3.conf @@ -0,0 +1,4 @@ +protected-mode no +port 63793 +dir . +logfile "/tmp/redis.log" From 355fb125cb447701ffb4316e4cddbc363264dc80 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 7 Jan 2020 15:18:16 -0500 Subject: [PATCH 02/41] redis events --- awx/main/dispatch/worker/__init__.py | 2 +- awx/main/dispatch/worker/base.py | 87 +++++++++++++++++++ .../commands/run_callback_receiver.py | 12 +-- awx/main/queue.py | 43 +-------- 4 files changed, 95 insertions(+), 49 deletions(-) diff --git a/awx/main/dispatch/worker/__init__.py b/awx/main/dispatch/worker/__init__.py index 009386914f..06d64c437c 100644 --- a/awx/main/dispatch/worker/__init__.py +++ b/awx/main/dispatch/worker/__init__.py @@ -1,3 +1,3 @@ -from .base import AWXConsumer, BaseWorker # noqa +from .base import AWXConsumer, AWXRedisConsumer, BaseWorker # noqa from .callback import CallbackBrokerWorker # noqa from .task import TaskWorker # noqa diff --git a/awx/main/dispatch/worker/base.py b/awx/main/dispatch/worker/base.py index 8c0a8a4b77..0a6bf4396b 100644 --- a/awx/main/dispatch/worker/base.py +++ b/awx/main/dispatch/worker/base.py @@ -5,12 +5,15 @@ import os import logging import signal import sys +import redis +import json from uuid import UUID from queue import Empty as QueueEmpty from django import db from kombu import Producer from kombu.mixins import ConsumerMixin +from django.conf import settings from awx.main.dispatch.pool import WorkerPool @@ -117,6 +120,90 @@ class AWXConsumer(ConsumerMixin): raise SystemExit() +class AWXRedisConsumer(object): + + def __init__(self, name, connection, worker, queues=[], pool=None): + self.should_stop = False + + self.name = name + self.connection = connection + self.total_messages = 0 + self.queues = queues + self.worker = worker + self.pool = pool + if pool is None: + self.pool = WorkerPool() + self.pool.init_workers(self.worker.work_loop) + + @property + def listening_on(self): + return f'listening on {self.queues}' + + ''' + def control(self, body, message): + logger.warn(body) + control = body.get('control') + if control in ('status', 'running'): + producer = Producer( + channel=self.connection, + routing_key=message.properties['reply_to'] + ) + if control == 'status': + msg = '\n'.join([self.listening_on, self.pool.debug()]) + elif control == 'running': + msg = [] + for worker in self.pool.workers: + worker.calculate_managed_tasks() + msg.extend(worker.managed_tasks.keys()) + producer.publish(msg) + elif control == 'reload': + for worker in self.pool.workers: + worker.quit() + else: + logger.error('unrecognized control message: {}'.format(control)) + message.ack() + ''' + + def process_task(self, body, message): + if 'control' in body: + try: + return self.control(body, message) + except Exception: + logger.exception("Exception handling control message:") + return + if len(self.pool): + if "uuid" in body and body['uuid']: + try: + queue = UUID(body['uuid']).int % len(self.pool) + except Exception: + queue = self.total_messages % len(self.pool) + else: + queue = self.total_messages % len(self.pool) + else: + queue = 0 + self.pool.write(queue, body) + self.total_messages += 1 + + def run(self, *args, **kwargs): + signal.signal(signal.SIGINT, self.stop) + signal.signal(signal.SIGTERM, self.stop) + self.worker.on_start() + + queue = redis.Redis.from_url(settings.BROKER_URL) + while True: + res = queue.blpop(self.queues) + res = json.loads(res[1]) + self.process_task(res, res) + if self.should_stop: + return + + def stop(self, signum, frame): + self.should_stop = True # this makes the kombu mixin stop consuming + logger.warn('received {}, stopping'.format(signame(signum))) + self.worker.on_stop() + raise SystemExit() + + class BaseWorker(object): def read(self, queue): diff --git a/awx/main/management/commands/run_callback_receiver.py b/awx/main/management/commands/run_callback_receiver.py index 58e311f2bb..269b01d98a 100644 --- a/awx/main/management/commands/run_callback_receiver.py +++ b/awx/main/management/commands/run_callback_receiver.py @@ -5,7 +5,7 @@ from django.conf import settings from django.core.management.base import BaseCommand from kombu import Exchange, Queue, Connection -from awx.main.dispatch.worker import AWXConsumer, CallbackBrokerWorker +from awx.main.dispatch.worker import AWXRedisConsumer, CallbackBrokerWorker class Command(BaseCommand): @@ -20,17 +20,11 @@ class Command(BaseCommand): with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: consumer = None try: - consumer = AWXConsumer( + consumer = AWXRedisConsumer( 'callback_receiver', conn, CallbackBrokerWorker(), - [ - Queue( - settings.CALLBACK_QUEUE, - Exchange(settings.CALLBACK_QUEUE, type='direct'), - routing_key=settings.CALLBACK_QUEUE - ) - ] + queues=[getattr(settings, 'CALLBACK_QUEUE', '')], ) consumer.run() except KeyboardInterrupt: diff --git a/awx/main/queue.py b/awx/main/queue.py index 3d8a8384eb..40cd58ce23 100644 --- a/awx/main/queue.py +++ b/awx/main/queue.py @@ -5,13 +5,13 @@ import json import logging import os +import redis # Django from django.conf import settings # Kombu from kombu import Exchange, Producer, Connection -from kombu.serialization import registry __all__ = ['CallbackQueueDispatcher'] @@ -27,48 +27,13 @@ class AnsibleJSONEncoder(json.JSONEncoder): return super(AnsibleJSONEncoder, self).default(o) -registry.register( - 'json-ansible', - lambda obj: json.dumps(obj, cls=AnsibleJSONEncoder), - lambda obj: json.loads(obj), - content_type='application/json', - content_encoding='utf-8' -) - - class CallbackQueueDispatcher(object): def __init__(self): self.callback_connection = getattr(settings, 'BROKER_URL', None) - self.callback_connection_options = getattr(settings, 'BROKER_TRANSPORT_OPTIONS', {}) - self.connection_queue = getattr(settings, 'CALLBACK_QUEUE', '') - self.connection = None - self.exchange = None + self.queue = getattr(settings, 'CALLBACK_QUEUE', '') self.logger = logging.getLogger('awx.main.queue.CallbackQueueDispatcher') + self.connection = redis.Redis.from_url(settings.BROKER_URL) def dispatch(self, obj): - if not self.callback_connection or not self.connection_queue: - return - active_pid = os.getpid() - for retry_count in range(4): - try: - if not hasattr(self, 'connection_pid'): - self.connection_pid = active_pid - if self.connection_pid != active_pid: - self.connection = None - if self.connection is None: - self.connection = Connection(self.callback_connection, transport_options=self.callback_connection_options) - self.exchange = Exchange(self.connection_queue, type='direct') - - producer = Producer(self.connection) - producer.publish(obj, - serializer='json-ansible', - compression='bzip2', - exchange=self.exchange, - declare=[self.exchange], - delivery_mode="transient", - routing_key=self.connection_queue) - return - except Exception as e: - self.logger.info('Publish Job Event Exception: %r, retry=%d', e, - retry_count, exc_info=True) + self.connection.rpush(self.queue, json.dumps(obj, cls=AnsibleJSONEncoder)) From 558e92806b0e692939faf113029ad322960c7369 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 20 Dec 2019 10:21:53 -0500 Subject: [PATCH 03/41] POC postgres broker --- awx/main/dispatch/publish.py | 35 ++-- awx/main/dispatch/worker/__init__.py | 1 + awx/main/dispatch/worker/basepg.py | 161 ++++++++++++++++++ awx/main/dispatch/worker/task.py | 4 +- .../management/commands/run_dispatcher.py | 16 +- 5 files changed, 180 insertions(+), 37 deletions(-) create mode 100644 awx/main/dispatch/worker/basepg.py diff --git a/awx/main/dispatch/publish.py b/awx/main/dispatch/publish.py index be64594ee3..cf94572e75 100644 --- a/awx/main/dispatch/publish.py +++ b/awx/main/dispatch/publish.py @@ -1,10 +1,14 @@ import inspect import logging import sys +import json from uuid import uuid4 +import psycopg2 from django.conf import settings -from kombu import Exchange, Producer, Connection, Queue, Consumer +from kombu import Exchange, Producer +from django.db import connection +from pgpubsub import PubSub logger = logging.getLogger('awx.main.dispatch') @@ -85,26 +89,15 @@ class task: if callable(queue): queue = queue() if not settings.IS_TESTING(sys.argv): - with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: - exchange = Exchange(queue, type=exchange_type or 'direct') - - # HACK: With Redis as the broker declaring an exchange isn't enough to create the queue - # Creating a Consumer _will_ create a queue so that publish will succeed. Note that we - # don't call consume() on the consumer so we don't actually eat any messages - Consumer(conn, queues=[Queue(queue, exchange, routing_key=queue)], accept=['json']) - producer = Producer(conn) - logger.debug('publish {}({}, queue={})'.format( - cls.name, - task_id, - queue - )) - producer.publish(obj, - serializer='json', - compression='bzip2', - exchange=exchange, - declare=[exchange], - delivery_mode="persistent", - routing_key=queue) + conf = settings.DATABASES['default'] + conn = psycopg2.connect(dbname=conf['NAME'], + host=conf['HOST'], + user=conf['USER'], + password=conf['PASSWORD']) + conn.set_session(autocommit=True) + logger.warn(f"Send message to queue {queue}") + pubsub = PubSub(conn) + pubsub.notify(queue, json.dumps(obj)) return (obj, queue) # If the object we're wrapping *is* a class (e.g., RunJob), return diff --git a/awx/main/dispatch/worker/__init__.py b/awx/main/dispatch/worker/__init__.py index 06d64c437c..5472f83579 100644 --- a/awx/main/dispatch/worker/__init__.py +++ b/awx/main/dispatch/worker/__init__.py @@ -1,3 +1,4 @@ from .base import AWXConsumer, AWXRedisConsumer, BaseWorker # noqa +from .basepg import AWXConsumerPG, BaseWorkerPG # noqa from .callback import CallbackBrokerWorker # noqa from .task import TaskWorker # noqa diff --git a/awx/main/dispatch/worker/basepg.py b/awx/main/dispatch/worker/basepg.py new file mode 100644 index 0000000000..7a35fc59d6 --- /dev/null +++ b/awx/main/dispatch/worker/basepg.py @@ -0,0 +1,161 @@ +# Copyright (c) 2018 Ansible by Red Hat +# All Rights Reserved. + +import os +import logging +import signal +import sys +import json +from uuid import UUID +from queue import Empty as QueueEmpty + +from django import db +from django.db import connection as pg_connection + +from pgpubsub import PubSub + +from awx.main.dispatch.pool import WorkerPool + +SHORT_CIRCUIT = False + +if 'run_callback_receiver' in sys.argv: + logger = logging.getLogger('awx.main.commands.run_callback_receiver') +else: + logger = logging.getLogger('awx.main.dispatch') + + +def signame(sig): + return dict( + (k, v) for v, k in signal.__dict__.items() + if v.startswith('SIG') and not v.startswith('SIG_') + )[sig] + + +class WorkerSignalHandler: + + def __init__(self): + self.kill_now = False + signal.signal(signal.SIGINT, self.exit_gracefully) + + def exit_gracefully(self, *args, **kwargs): + self.kill_now = True + + +class AWXConsumerPG(object): + + def __init__(self, name, connection, worker, queues=[], pool=None): + self.name = name + self.connection = pg_connection + self.total_messages = 0 + self.queues = queues + self.worker = worker + self.pool = pool + # TODO, maybe get new connection and reconnect periodically + self.pubsub = PubSub(pg_connection.cursor().connection) + if pool is None: + self.pool = WorkerPool() + self.pool.init_workers(self.worker.work_loop) + + @property + def listening_on(self): + return 'listening on {}'.format([f'{q}' for q in self.queues]) + + def control(self, body, message): + logger.warn(body) + control = body.get('control') + if control in ('status', 'running'): + if control == 'status': + msg = '\n'.join([self.listening_on, self.pool.debug()]) + elif control == 'running': + msg = [] + for worker in self.pool.workers: + worker.calculate_managed_tasks() + msg.extend(worker.managed_tasks.keys()) + self.pubsub.notify(message.properties['reply_to'], msg) + elif control == 'reload': + for worker in self.pool.workers: + worker.quit() + else: + logger.error('unrecognized control message: {}'.format(control)) + + def process_task(self, body, message): + if SHORT_CIRCUIT or 'control' in body: + try: + return self.control(body, message) + except Exception: + logger.exception("Exception handling control message:") + return + if len(self.pool): + if "uuid" in body and body['uuid']: + try: + queue = UUID(body['uuid']).int % len(self.pool) + except Exception: + queue = self.total_messages % len(self.pool) + else: + queue = self.total_messages % len(self.pool) + else: + queue = 0 + self.pool.write(queue, body) + self.total_messages += 1 + + def run(self, *args, **kwargs): + signal.signal(signal.SIGINT, self.stop) + signal.signal(signal.SIGTERM, self.stop) + self.worker.on_start() + + logger.warn(f"Running worker {self.name} listening to queues {self.queues}") + self.pubsub = PubSub(pg_connection.cursor().connection) + for queue in self.queues: + self.pubsub.listen(queue) + for e in self.pubsub.events(): + logger.warn(f"Processing task {e}") + self.process_task(json.loads(e.payload), e) + + def stop(self, signum, frame): + logger.warn('received {}, stopping'.format(signame(signum))) + for queue in self.queues: + self.pubsub.unlisten(queue) + self.worker.on_stop() + raise SystemExit() + + +class BaseWorkerPG(object): + + def work_loop(self, queue, finished, idx, *args): + ppid = os.getppid() + signal_handler = WorkerSignalHandler() + while not signal_handler.kill_now: + # if the parent PID changes, this process has been orphaned + # via e.g., segfault or sigkill, we should exit too + if os.getppid() != ppid: + break + try: + body = queue.get(block=True, timeout=1) + if body == 'QUIT': + break + except QueueEmpty: + continue + except Exception as e: + logger.error("Exception on worker {}, restarting: ".format(idx) + str(e)) + continue + try: + for conn in db.connections.all(): + # If the database connection has a hiccup during the prior message, close it + # so we can establish a new connection + conn.close_if_unusable_or_obsolete() + self.perform_work(body, *args) + finally: + if 'uuid' in body: + uuid = body['uuid'] + logger.debug('task {} is finished'.format(uuid)) + finished.put(uuid) + logger.warn('worker exiting gracefully pid:{}'.format(os.getpid())) + + def perform_work(self, body): + raise NotImplementedError() + + def on_start(self): + pass + + def on_stop(self): + pass diff --git a/awx/main/dispatch/worker/task.py b/awx/main/dispatch/worker/task.py index 7e7437d445..80c1907fc1 100644 --- a/awx/main/dispatch/worker/task.py +++ b/awx/main/dispatch/worker/task.py @@ -8,12 +8,12 @@ from kubernetes.config import kube_config from awx.main.tasks import dispatch_startup, inform_cluster_of_shutdown -from .base import BaseWorker +from .basepg import BaseWorkerPG logger = logging.getLogger('awx.main.dispatch') -class TaskWorker(BaseWorker): +class TaskWorker(BaseWorkerPG): ''' A worker implementation that deserializes task messages and runs native Python code. diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py index 7e69897687..ea6098db4c 100644 --- a/awx/main/management/commands/run_dispatcher.py +++ b/awx/main/management/commands/run_dispatcher.py @@ -64,20 +64,8 @@ class Command(BaseCommand): AWXProxyHandler.disable() with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: try: - bcast = 'tower_broadcast_all' - queues = [ - Queue(q, Exchange(q), routing_key=q) - for q in (settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()]) - ] - queues.append( - Queue( - construct_bcast_queue_name(bcast), - exchange=Exchange(bcast, type='fanout'), - routing_key=bcast, - reply=True - ) - ) - consumer = AWXConsumer( + queues = ['tower_broadcast_all'] + settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()] + consumer = AWXConsumerPG( 'dispatcher', conn, TaskWorker(), From 2a2c34f56779e7ccd8337141efc3bef8f6695c3e Mon Sep 17 00:00:00 2001 From: chris meyers Date: Wed, 8 Jan 2020 15:44:04 -0500 Subject: [PATCH 04/41] combine all the broker replacement pieces * local redis for event processing * postgres for message broker * redis for websockets --- awx/main/dispatch/__init__.py | 80 +++++++++ awx/main/dispatch/control.py | 53 +++--- awx/main/dispatch/publish.py | 24 +-- awx/main/dispatch/worker/__init__.py | 3 +- awx/main/dispatch/worker/base.py | 145 +++++----------- awx/main/dispatch/worker/basepg.py | 161 ------------------ awx/main/dispatch/worker/task.py | 4 +- .../commands/run_callback_receiver.py | 4 +- .../management/commands/run_dispatcher.py | 29 ++-- awx/main/queue.py | 1 - awx/main/tasks.py | 6 +- awx/settings/defaults.py | 2 +- awx/settings/local_settings.py.docker_compose | 6 +- tools/docker-compose-cluster.yml | 13 +- 14 files changed, 188 insertions(+), 343 deletions(-) delete mode 100644 awx/main/dispatch/worker/basepg.py diff --git a/awx/main/dispatch/__init__.py b/awx/main/dispatch/__init__.py index 50f912427e..d97919dada 100644 --- a/awx/main/dispatch/__init__.py +++ b/awx/main/dispatch/__init__.py @@ -1,5 +1,85 @@ +import psycopg2 +import select +import sys +import logging + +from contextlib import contextmanager + from django.conf import settings +NOT_READY = ([], [], []) +if 'run_callback_receiver' in sys.argv: + logger = logging.getLogger('awx.main.commands.run_callback_receiver') +else: + logger = logging.getLogger('awx.main.dispatch') + + def get_local_queuename(): return settings.CLUSTER_HOST_ID + + +class PubSub(object): + def __init__(self, conn): + assert conn.autocommit, "Connection must be in autocommit mode." + self.conn = conn + + def listen(self, channel): + with self.conn.cursor() as cur: + cur.execute('LISTEN "%s";' % channel) + + def unlisten(self, channel): + with self.conn.cursor() as cur: + cur.execute('UNLISTEN "%s";' % channel) + + def notify(self, channel, payload): + with self.conn.cursor() as cur: + cur.execute('SELECT pg_notify(%s, %s);', (channel, payload)) + + def get_event(self, select_timeout=0): + # poll the connection, then return one event, if we have one. Else + # return None. + select.select([self.conn], [], [], select_timeout) + self.conn.poll() + if self.conn.notifies: + return self.conn.notifies.pop(0) + + def get_events(self, select_timeout=0): + # Poll the connection and return all events, if there are any. Else + # return None. + select.select([self.conn], [], [], select_timeout) # redundant? + self.conn.poll() + events = [] + while self.conn.notifies: + events.append(self.conn.notifies.pop(0)) + if events: + return events + + def events(self, select_timeout=5, yield_timeouts=False): + while True: + if select.select([self.conn], [], [], select_timeout) == NOT_READY: + if yield_timeouts: + yield None + else: + self.conn.poll() + while self.conn.notifies: + yield self.conn.notifies.pop(0) + + def close(self): + self.conn.close() + + +@contextmanager +def pg_bus_conn(): + conf = settings.DATABASES['default'] + conn = psycopg2.connect(dbname=conf['NAME'], + host=conf['HOST'], + user=conf['USER'], + password=conf['PASSWORD']) + # Django connection.cursor().connection doesn't have autocommit=True on + conn.set_session(autocommit=True) + pubsub = PubSub(conn) + yield pubsub + conn.close() + + diff --git a/awx/main/dispatch/control.py b/awx/main/dispatch/control.py index f938aab6b5..a584c9dfe5 100644 --- a/awx/main/dispatch/control.py +++ b/awx/main/dispatch/control.py @@ -1,11 +1,14 @@ import logging import socket - -from django.conf import settings +import string +import random +import json from awx.main.dispatch import get_local_queuename from kombu import Queue, Exchange, Producer, Consumer, Connection +from . import pg_bus_conn + logger = logging.getLogger('awx.main.dispatch') @@ -19,15 +22,10 @@ class Control(object): raise RuntimeError('{} must be in {}'.format(service, self.services)) self.service = service self.queuename = host or get_local_queuename() - self.queue = Queue(self.queuename, Exchange(self.queuename), routing_key=self.queuename) def publish(self, msg, conn, **kwargs): - producer = Producer( - exchange=self.queue.exchange, - channel=conn, - routing_key=self.queuename - ) - producer.publish(msg, expiration=5, **kwargs) + # TODO: delete this method?? + raise RuntimeError("Publish called?!") def status(self, *args, **kwargs): return self.control_with_reply('status', *args, **kwargs) @@ -35,24 +33,29 @@ class Control(object): def running(self, *args, **kwargs): return self.control_with_reply('running', *args, **kwargs) + @classmethod + def generate_reply_queue_name(cls): + letters = string.ascii_lowercase + return 'reply_to_{}'.format(''.join(random.choice(letters) for i in range(8))) + def control_with_reply(self, command, timeout=5): logger.warn('checking {} {} for {}'.format(self.service, command, self.queuename)) - reply_queue = Queue(name="amq.rabbitmq.reply-to") + reply_queue = Control.generate_reply_queue_name() self.result = None - with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: - with Consumer(conn, reply_queue, callbacks=[self.process_message], no_ack=True): - self.publish({'control': command}, conn, reply_to='amq.rabbitmq.reply-to') - try: - conn.drain_events(timeout=timeout) - except socket.timeout: - logger.error('{} did not reply within {}s'.format(self.service, timeout)) - raise - return self.result + + with pg_bus_conn() as conn: + conn.listen(reply_queue) + conn.notify(self.queuename, + json.dumps({'control': command, 'reply_to': reply_queue})) + + for reply in conn.events(select_timeout=timeout, yield_timeouts=True): + if reply is None: + logger.error(f'{self.service} did not reply within {timeout}s') + raise RuntimeError("{self.service} did not reply within {timeout}s") + break + + return json.loads(reply.payload) def control(self, msg, **kwargs): - with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: - self.publish(msg, conn) - - def process_message(self, body, message): - self.result = body - message.ack() + with pg_bus_conn() as conn: + conn.notify(self.queuename, json.dumps(msg)) diff --git a/awx/main/dispatch/publish.py b/awx/main/dispatch/publish.py index cf94572e75..fe77bd4c37 100644 --- a/awx/main/dispatch/publish.py +++ b/awx/main/dispatch/publish.py @@ -2,14 +2,13 @@ import inspect import logging import sys import json +import re from uuid import uuid4 -import psycopg2 from django.conf import settings -from kombu import Exchange, Producer from django.db import connection -from pgpubsub import PubSub +from . import pg_bus_conn logger = logging.getLogger('awx.main.dispatch') @@ -42,24 +41,22 @@ class task: add.apply_async([1, 1]) Adder.apply_async([1, 1]) - # Tasks can also define a specific target queue or exchange type: + # Tasks can also define a specific target queue or use the special fan-out queue tower_broadcast: @task(queue='slow-tasks') def snooze(): time.sleep(10) - @task(queue='tower_broadcast', exchange_type='fanout') + @task(queue='tower_broadcast') def announce(): print("Run this everywhere!") """ - def __init__(self, queue=None, exchange_type=None): + def __init__(self, queue=None): self.queue = queue - self.exchange_type = exchange_type def __call__(self, fn=None): queue = self.queue - exchange_type = self.exchange_type class PublisherMixin(object): @@ -89,15 +86,8 @@ class task: if callable(queue): queue = queue() if not settings.IS_TESTING(sys.argv): - conf = settings.DATABASES['default'] - conn = psycopg2.connect(dbname=conf['NAME'], - host=conf['HOST'], - user=conf['USER'], - password=conf['PASSWORD']) - conn.set_session(autocommit=True) - logger.warn(f"Send message to queue {queue}") - pubsub = PubSub(conn) - pubsub.notify(queue, json.dumps(obj)) + with pg_bus_conn() as conn: + conn.notify(queue, json.dumps(obj)) return (obj, queue) # If the object we're wrapping *is* a class (e.g., RunJob), return diff --git a/awx/main/dispatch/worker/__init__.py b/awx/main/dispatch/worker/__init__.py index 5472f83579..6fe8f64608 100644 --- a/awx/main/dispatch/worker/__init__.py +++ b/awx/main/dispatch/worker/__init__.py @@ -1,4 +1,3 @@ -from .base import AWXConsumer, AWXRedisConsumer, BaseWorker # noqa -from .basepg import AWXConsumerPG, BaseWorkerPG # noqa +from .base import AWXConsumerRedis, AWXConsumerPG, BaseWorker # noqa from .callback import CallbackBrokerWorker # noqa from .task import TaskWorker # noqa diff --git a/awx/main/dispatch/worker/base.py b/awx/main/dispatch/worker/base.py index 0a6bf4396b..b90b41ce1c 100644 --- a/awx/main/dispatch/worker/base.py +++ b/awx/main/dispatch/worker/base.py @@ -7,6 +7,8 @@ import signal import sys import redis import json +import re +import psycopg2 from uuid import UUID from queue import Empty as QueueEmpty @@ -16,6 +18,7 @@ from kombu.mixins import ConsumerMixin from django.conf import settings from awx.main.dispatch.pool import WorkerPool +from awx.main.dispatch import pg_bus_conn if 'run_callback_receiver' in sys.argv: logger = logging.getLogger('awx.main.commands.run_callback_receiver') @@ -40,88 +43,7 @@ class WorkerSignalHandler: self.kill_now = True -class AWXConsumer(ConsumerMixin): - - def __init__(self, name, connection, worker, queues=[], pool=None): - self.connection = connection - self.total_messages = 0 - self.queues = queues - self.worker = worker - self.pool = pool - if pool is None: - self.pool = WorkerPool() - self.pool.init_workers(self.worker.work_loop) - - def get_consumers(self, Consumer, channel): - logger.debug(self.listening_on) - return [Consumer(queues=self.queues, accept=['json'], - callbacks=[self.process_task])] - - @property - def listening_on(self): - return 'listening on {}'.format([ - '{} [{}]'.format(q.name, q.exchange.type) for q in self.queues - ]) - - def control(self, body, message): - logger.warn('Consumer received control message {}'.format(body)) - control = body.get('control') - if control in ('status', 'running'): - producer = Producer( - channel=self.connection, - routing_key=message.properties['reply_to'] - ) - if control == 'status': - msg = '\n'.join([self.listening_on, self.pool.debug()]) - elif control == 'running': - msg = [] - for worker in self.pool.workers: - worker.calculate_managed_tasks() - msg.extend(worker.managed_tasks.keys()) - producer.publish(msg) - elif control == 'reload': - for worker in self.pool.workers: - worker.quit() - else: - logger.error('unrecognized control message: {}'.format(control)) - message.ack() - - def process_task(self, body, message): - if 'control' in body: - try: - return self.control(body, message) - except Exception: - logger.exception("Exception handling control message:") - return - if len(self.pool): - if "uuid" in body and body['uuid']: - try: - queue = UUID(body['uuid']).int % len(self.pool) - except Exception: - queue = self.total_messages % len(self.pool) - else: - queue = self.total_messages % len(self.pool) - else: - queue = 0 - self.pool.write(queue, body) - self.total_messages += 1 - message.ack() - - def run(self, *args, **kwargs): - signal.signal(signal.SIGINT, self.stop) - signal.signal(signal.SIGTERM, self.stop) - self.worker.on_start() - super(AWXConsumer, self).run(*args, **kwargs) - - def stop(self, signum, frame): - self.should_stop = True # this makes the kombu mixin stop consuming - logger.warn('received {}, stopping'.format(signame(signum))) - self.worker.on_stop() - raise SystemExit() - - -class AWXRedisConsumer(object): - +class AWXConsumerBase(object): def __init__(self, name, connection, worker, queues=[], pool=None): self.should_stop = False @@ -139,15 +61,11 @@ class AWXRedisConsumer(object): def listening_on(self): return f'listening on {self.queues}' - ''' - def control(self, body, message): + def control(self, body): logger.warn(body) control = body.get('control') if control in ('status', 'running'): - producer = Producer( - channel=self.connection, - routing_key=message.properties['reply_to'] - ) + reply_queue = body['reply_to'] if control == 'status': msg = '\n'.join([self.listening_on, self.pool.debug()]) elif control == 'running': @@ -155,21 +73,21 @@ class AWXRedisConsumer(object): for worker in self.pool.workers: worker.calculate_managed_tasks() msg.extend(worker.managed_tasks.keys()) - producer.publish(msg) + + with pg_bus_conn() as conn: + conn.notify(reply_queue, json.dumps(msg)) elif control == 'reload': for worker in self.pool.workers: worker.quit() else: logger.error('unrecognized control message: {}'.format(control)) - message.ack() - ''' - def process_task(self, body, message): + def process_task(self, body): if 'control' in body: try: - return self.control(body, message) + return self.control(body) except Exception: - logger.exception("Exception handling control message:") + logger.exception(f"Exception handling control message: {body}") return if len(self.pool): if "uuid" in body and body['uuid']: @@ -189,13 +107,7 @@ class AWXRedisConsumer(object): signal.signal(signal.SIGTERM, self.stop) self.worker.on_start() - queue = redis.Redis.from_url(settings.BROKER_URL) - while True: - res = queue.blpop(self.queues) - res = json.loads(res[1]) - self.process_task(res, res) - if self.should_stop: - return + # Child should implement other things here def stop(self, signum, frame): self.should_stop = True # this makes the kombu mixin stop consuming @@ -204,6 +116,37 @@ class AWXRedisConsumer(object): raise SystemExit() +class AWXConsumerRedis(AWXConsumerBase): + def run(self, *args, **kwargs): + super(AWXConsumerRedis, self).run(*args, **kwargs) + + queue = redis.Redis.from_url(settings.BROKER_URL) + while True: + res = queue.blpop(self.queues) + res = json.loads(res[1]) + self.process_task(res) + if self.should_stop: + return + + +class AWXConsumerPG(AWXConsumerBase): + def run(self, *args, **kwargs): + super(AWXConsumerPG, self).run(*args, **kwargs) + + logger.warn(f"Running worker {self.name} listening to queues {self.queues}") + + while True: + try: + with pg_bus_conn() as conn: + for queue in self.queues: + conn.listen(queue) + for e in conn.events(): + self.process_task(json.loads(e.payload)) + except psycopg2.InterfaceError: + logger.warn("Stale Postgres message bus connection, reconnecting") + continue + + class BaseWorker(object): def read(self, queue): diff --git a/awx/main/dispatch/worker/basepg.py b/awx/main/dispatch/worker/basepg.py deleted file mode 100644 index 7a35fc59d6..0000000000 --- a/awx/main/dispatch/worker/basepg.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright (c) 2018 Ansible by Red Hat -# All Rights Reserved. - -import os -import logging -import signal -import sys -import json -from uuid import UUID -from queue import Empty as QueueEmpty - -from django import db -from django.db import connection as pg_connection - -from pgpubsub import PubSub - -from awx.main.dispatch.pool import WorkerPool - -SHORT_CIRCUIT = False - -if 'run_callback_receiver' in sys.argv: - logger = logging.getLogger('awx.main.commands.run_callback_receiver') -else: - logger = logging.getLogger('awx.main.dispatch') - - -def signame(sig): - return dict( - (k, v) for v, k in signal.__dict__.items() - if v.startswith('SIG') and not v.startswith('SIG_') - )[sig] - - -class WorkerSignalHandler: - - def __init__(self): - self.kill_now = False - signal.signal(signal.SIGINT, self.exit_gracefully) - - def exit_gracefully(self, *args, **kwargs): - self.kill_now = True - - -class AWXConsumerPG(object): - - def __init__(self, name, connection, worker, queues=[], pool=None): - self.name = name - self.connection = pg_connection - self.total_messages = 0 - self.queues = queues - self.worker = worker - self.pool = pool - # TODO, maybe get new connection and reconnect periodically - self.pubsub = PubSub(pg_connection.cursor().connection) - if pool is None: - self.pool = WorkerPool() - self.pool.init_workers(self.worker.work_loop) - - @property - def listening_on(self): - return 'listening on {}'.format([f'{q}' for q in self.queues]) - - def control(self, body, message): - logger.warn(body) - control = body.get('control') - if control in ('status', 'running'): - if control == 'status': - msg = '\n'.join([self.listening_on, self.pool.debug()]) - elif control == 'running': - msg = [] - for worker in self.pool.workers: - worker.calculate_managed_tasks() - msg.extend(worker.managed_tasks.keys()) - self.pubsub.notify(message.properties['reply_to'], msg) - elif control == 'reload': - for worker in self.pool.workers: - worker.quit() - else: - logger.error('unrecognized control message: {}'.format(control)) - - def process_task(self, body, message): - if SHORT_CIRCUIT or 'control' in body: - try: - return self.control(body, message) - except Exception: - logger.exception("Exception handling control message:") - return - if len(self.pool): - if "uuid" in body and body['uuid']: - try: - queue = UUID(body['uuid']).int % len(self.pool) - except Exception: - queue = self.total_messages % len(self.pool) - else: - queue = self.total_messages % len(self.pool) - else: - queue = 0 - self.pool.write(queue, body) - self.total_messages += 1 - - def run(self, *args, **kwargs): - signal.signal(signal.SIGINT, self.stop) - signal.signal(signal.SIGTERM, self.stop) - self.worker.on_start() - - logger.warn(f"Running worker {self.name} listening to queues {self.queues}") - self.pubsub = PubSub(pg_connection.cursor().connection) - for queue in self.queues: - self.pubsub.listen(queue) - for e in self.pubsub.events(): - logger.warn(f"Processing task {e}") - self.process_task(json.loads(e.payload), e) - - def stop(self, signum, frame): - logger.warn('received {}, stopping'.format(signame(signum))) - for queue in self.queues: - self.pubsub.unlisten(queue) - self.worker.on_stop() - raise SystemExit() - - -class BaseWorkerPG(object): - - def work_loop(self, queue, finished, idx, *args): - ppid = os.getppid() - signal_handler = WorkerSignalHandler() - while not signal_handler.kill_now: - # if the parent PID changes, this process has been orphaned - # via e.g., segfault or sigkill, we should exit too - if os.getppid() != ppid: - break - try: - body = queue.get(block=True, timeout=1) - if body == 'QUIT': - break - except QueueEmpty: - continue - except Exception as e: - logger.error("Exception on worker {}, restarting: ".format(idx) + str(e)) - continue - try: - for conn in db.connections.all(): - # If the database connection has a hiccup during the prior message, close it - # so we can establish a new connection - conn.close_if_unusable_or_obsolete() - self.perform_work(body, *args) - finally: - if 'uuid' in body: - uuid = body['uuid'] - logger.debug('task {} is finished'.format(uuid)) - finished.put(uuid) - logger.warn('worker exiting gracefully pid:{}'.format(os.getpid())) - - def perform_work(self, body): - raise NotImplementedError() - - def on_start(self): - pass - - def on_stop(self): - pass diff --git a/awx/main/dispatch/worker/task.py b/awx/main/dispatch/worker/task.py index 80c1907fc1..7e7437d445 100644 --- a/awx/main/dispatch/worker/task.py +++ b/awx/main/dispatch/worker/task.py @@ -8,12 +8,12 @@ from kubernetes.config import kube_config from awx.main.tasks import dispatch_startup, inform_cluster_of_shutdown -from .basepg import BaseWorkerPG +from .base import BaseWorker logger = logging.getLogger('awx.main.dispatch') -class TaskWorker(BaseWorkerPG): +class TaskWorker(BaseWorker): ''' A worker implementation that deserializes task messages and runs native Python code. diff --git a/awx/main/management/commands/run_callback_receiver.py b/awx/main/management/commands/run_callback_receiver.py index 269b01d98a..f1c1aed0c3 100644 --- a/awx/main/management/commands/run_callback_receiver.py +++ b/awx/main/management/commands/run_callback_receiver.py @@ -5,7 +5,7 @@ from django.conf import settings from django.core.management.base import BaseCommand from kombu import Exchange, Queue, Connection -from awx.main.dispatch.worker import AWXRedisConsumer, CallbackBrokerWorker +from awx.main.dispatch.worker import AWXConsumerRedis, CallbackBrokerWorker class Command(BaseCommand): @@ -20,7 +20,7 @@ class Command(BaseCommand): with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: consumer = None try: - consumer = AWXRedisConsumer( + consumer = AWXConsumerRedis( 'callback_receiver', conn, CallbackBrokerWorker(), diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py index ea6098db4c..c62dc6c732 100644 --- a/awx/main/management/commands/run_dispatcher.py +++ b/awx/main/management/commands/run_dispatcher.py @@ -62,18 +62,17 @@ class Command(BaseCommand): # in cpython itself: # https://bugs.python.org/issue37429 AWXProxyHandler.disable() - with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: - try: - queues = ['tower_broadcast_all'] + settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()] - consumer = AWXConsumerPG( - 'dispatcher', - conn, - TaskWorker(), - queues, - AutoscalePool(min_workers=4) - ) - consumer.run() - except KeyboardInterrupt: - logger.debug('Terminating Task Dispatcher') - if consumer: - consumer.stop() + try: + queues = ['tower_broadcast_all'] + settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()] + consumer = AWXConsumerPG( + 'dispatcher', + None, + TaskWorker(), + queues, + AutoscalePool(min_workers=4) + ) + consumer.run() + except KeyboardInterrupt: + logger.debug('Terminating Task Dispatcher') + if consumer: + consumer.stop() diff --git a/awx/main/queue.py b/awx/main/queue.py index 40cd58ce23..8d4fffdf87 100644 --- a/awx/main/queue.py +++ b/awx/main/queue.py @@ -30,7 +30,6 @@ class AnsibleJSONEncoder(json.JSONEncoder): class CallbackQueueDispatcher(object): def __init__(self): - self.callback_connection = getattr(settings, 'BROKER_URL', None) self.queue = getattr(settings, 'CALLBACK_QUEUE', '') self.logger = logging.getLogger('awx.main.queue.CallbackQueueDispatcher') self.connection = redis.Redis.from_url(settings.BROKER_URL) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 292ae0f17e..e45f34cd66 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -264,7 +264,7 @@ def apply_cluster_membership_policies(): logger.debug('Cluster policy computation finished in {} seconds'.format(time.time() - started_compute)) -@task(queue='tower_broadcast_all', exchange_type='fanout') +@task(queue='tower_broadcast_all') def handle_setting_changes(setting_keys): orig_len = len(setting_keys) for i in range(orig_len): @@ -275,7 +275,7 @@ def handle_setting_changes(setting_keys): cache.delete_many(cache_keys) -@task(queue='tower_broadcast_all', exchange_type='fanout') +@task(queue='tower_broadcast_all') def delete_project_files(project_path): # TODO: possibly implement some retry logic lock_file = project_path + '.lock' @@ -293,7 +293,7 @@ def delete_project_files(project_path): logger.exception('Could not remove lock file {}'.format(lock_file)) -@task(queue='tower_broadcast_all', exchange_type='fanout') +@task(queue='tower_broadcast_all') def profile_sql(threshold=1, minutes=1): if threshold == 0: cache.delete('awx-profile-sql-threshold') diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 252e39d708..f6ad6f6d7c 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -421,7 +421,7 @@ os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199') BROKER_DURABILITY = True BROKER_POOL_LIMIT = None -BROKER_URL = 'redis://localhost:6379;' +BROKER_URL = 'redis://localhost:6379' BROKER_TRANSPORT_OPTIONS = {} CELERY_DEFAULT_QUEUE = 'awx_private_queue' CELERYBEAT_SCHEDULE = { diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index 8ebaa8747e..e2ae322e3d 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -14,6 +14,7 @@ import os import urllib.parse import sys +from urllib import parse # Enable the following lines and install the browser extension to use Django debug toolbar # if your deployment method is not VMWare of Docker-for-Mac you may @@ -53,12 +54,13 @@ if "pytest" in sys.modules: # Default to "just works" for single tower docker BROKER_URL = os.environ.get('BROKER_URL', "redis://redis_1:6379") +redis_parts = parse.urlparse(BROKER_URL) + CHANNEL_LAYERS = { "default": { "BACKEND": "awx.main.channels.RedisGroupBroadcastChannelLayer", "CONFIG": { - "hosts": [(os.environ.get('REDIS_HOST', 'redis_1'), - int(os.environ.get('REDIS_PORT', 6379)))], + "hosts": [(redis_parts.hostname, redis_parts.port)] }, }, } diff --git a/tools/docker-compose-cluster.yml b/tools/docker-compose-cluster.yml index caa2a7540f..6a7f393e52 100644 --- a/tools/docker-compose-cluster.yml +++ b/tools/docker-compose-cluster.yml @@ -24,10 +24,7 @@ services: #entrypoint: ["bash"] environment: CURRENT_UID: - # BROKER_URL will go away when we use postgres as our message broker BROKER_URL: "redis://redis_1:63791" - REDIS_HOST: redis_1 - REDIS_PORT: 63791 SDB_HOST: 0.0.0.0 SDB_PORT: 5899 AWX_GROUP_QUEUES: alpha,tower @@ -47,10 +44,7 @@ services: working_dir: "/awx_devel" environment: CURRENT_UID: - # BROKER_URL will go away when we use postgres as our message broker - BROKER_URL: "redis://redis_1:63791" - REDIS_HOST: redis_2 - REDIS_PORT: 63792 + BROKER_URL: "redis://redis_2:63792" SDB_HOST: 0.0.0.0 SDB_PORT: 7899 AWX_GROUP_QUEUES: bravo,tower @@ -69,10 +63,7 @@ services: working_dir: "/awx_devel" environment: CURRENT_UID: - # BROKER_URL will go away when we use postgres as our message broker - BROKER_URL: "redis://redis_1:63791" - REDIS_HOST: redis_3 - REDIS_PORT: 63793 + BROKER_URL: "redis://redis_3:63793" SDB_HOST: 0.0.0.0 SDB_PORT: 8899 AWX_GROUP_QUEUES: charlie,tower From 3fec69799cb37854da8e6138041bb1ec4949ae95 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Mon, 13 Jan 2020 15:01:29 -0500 Subject: [PATCH 05/41] fix websocket job subscription access control --- awx/main/consumers.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/awx/main/consumers.py b/awx/main/consumers.py index c81085c988..bc7aa47089 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -140,7 +140,7 @@ class EventConsumer(AsyncJsonWebsocketConsumer): await self.close() @database_sync_to_async - def user_can_see_object_id(self, user_access): + def user_can_see_object_id(self, user_access, oid): return user_access.get_queryset().filter(pk=oid).exists() async def receive_json(self, data): @@ -169,17 +169,16 @@ class EventConsumer(AsyncJsonWebsocketConsumer): access_cls = consumer_access(group_name) if access_cls is not None: user_access = access_cls(user) - if not self.user_can_see_object_id(user_access): + if not await self.user_can_see_object_id(user_access, oid): await self.send_json({"error": "access denied to channel {0} for resource id {1}".format(group_name, oid)}) continue - new_groups.add(name) else: if group_name == BROADCAST_GROUP: logger.warn("Non-priveleged client asked to join broadcast group!") return - new_groups.add(name) + new_groups.add(group_name) old_groups = current_groups - new_groups for group_name in old_groups: From 50b56aa8cb0c03bfc41139e5cbe7cc8836f14f11 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 14 Jan 2020 11:23:03 -0500 Subject: [PATCH 06/41] autobahn 20.1.2 released an hour ago * 20.1.1 no longer available on pypi --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index d778a73922..8366788ebc 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -7,7 +7,7 @@ ansiconv==1.0.0 asgiref==3.2.3 # via channels, channels-redis, daphne async-timeout==3.0.1 # via aiohttp, aioredis attrs==19.3.0 # via aiohttp, automat, jsonschema, service-identity, twisted -autobahn==20.1.1 # via daphne +autobahn==20.1.2 # via daphne automat==0.8.0 # via twisted azure-common==1.1.24 # via azure-keyvault azure-keyvault==1.1.0 From dc6c353ecd78146ca2965edb24431f56baccdb6a Mon Sep 17 00:00:00 2001 From: chris meyers Date: Thu, 16 Jan 2020 14:26:16 -0500 Subject: [PATCH 07/41] remove support for multi-reader dispatch queue * Under the new postgres backed notify/listen message queue, this never actually worked. Without using the database to store state, we can not provide a at-most-once delivery mechanism w/ multi-readers. * With this change, work is done ONLY on the node that requested for the work to be done. Under rabbitmq, the node that was first to get the message off the queue would do the work; presumably the least busy node. --- awx/main/dispatch/publish.py | 9 ++++-- .../management/commands/run_dispatcher.py | 2 +- awx/main/models/unified_jobs.py | 3 +- awx/main/scheduler/tasks.py | 3 +- awx/main/tasks.py | 30 +++++++++---------- awx/settings/defaults.py | 9 ------ 6 files changed, 26 insertions(+), 30 deletions(-) diff --git a/awx/main/dispatch/publish.py b/awx/main/dispatch/publish.py index fe77bd4c37..02fca647f6 100644 --- a/awx/main/dispatch/publish.py +++ b/awx/main/dispatch/publish.py @@ -8,7 +8,7 @@ from uuid import uuid4 from django.conf import settings from django.db import connection -from . import pg_bus_conn +from . import pg_bus_conn, get_local_queuename logger = logging.getLogger('awx.main.dispatch') @@ -73,9 +73,12 @@ class task: kwargs = kwargs or {} queue = ( queue or - getattr(cls.queue, 'im_func', cls.queue) or - settings.CELERY_DEFAULT_QUEUE + getattr(cls.queue, 'im_func', cls.queue) ) + if not queue: + msg = f'{cls.name}: Queue value required and may not me None' + logger.error(msg) + raise ValueError(msg) obj = { 'uuid': task_id, 'args': args, diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py index c62dc6c732..b678797026 100644 --- a/awx/main/management/commands/run_dispatcher.py +++ b/awx/main/management/commands/run_dispatcher.py @@ -63,7 +63,7 @@ class Command(BaseCommand): # https://bugs.python.org/issue37429 AWXProxyHandler.disable() try: - queues = ['tower_broadcast_all'] + settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()] + queues = ['tower_broadcast_all', get_local_queuename()] consumer = AWXConsumerPG( 'dispatcher', None, diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 253eb7b57f..6ab9ee6066 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -36,6 +36,7 @@ from awx.main.models.base import ( NotificationFieldsModel, prevent_search ) +from awx.main.dispatch import get_local_queuename from awx.main.dispatch.control import Control as ControlDispatcher from awx.main.registrar import activity_stream_registrar from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin @@ -1466,7 +1467,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique return r def get_queue_name(self): - return self.controller_node or self.execution_node or settings.CELERY_DEFAULT_QUEUE + return self.controller_node or self.execution_node or get_local_queuename() def is_isolated(self): return bool(self.controller_node) diff --git a/awx/main/scheduler/tasks.py b/awx/main/scheduler/tasks.py index c0d3dd842e..7da6a305a9 100644 --- a/awx/main/scheduler/tasks.py +++ b/awx/main/scheduler/tasks.py @@ -5,11 +5,12 @@ import logging # AWX from awx.main.scheduler import TaskManager from awx.main.dispatch.publish import task +from awx.main.dispatch import get_local_queuename logger = logging.getLogger('awx.main.scheduler') -@task() +@task(queue=get_local_queuename) def run_task_manager(): logger.debug("Running Tower task manager.") TaskManager().schedule() diff --git a/awx/main/tasks.py b/awx/main/tasks.py index e45f34cd66..4ab008cd31 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -151,7 +151,7 @@ def inform_cluster_of_shutdown(): logger.exception('Encountered problem with normal shutdown signal.') -@task() +@task(queue=get_local_queuename) def apply_cluster_membership_policies(): started_waiting = time.time() with advisory_lock('cluster_policy_lock', wait=True): @@ -307,7 +307,7 @@ def profile_sql(threshold=1, minutes=1): logger.error('SQL QUERIES >={}s ENABLED FOR {} MINUTE(S)'.format(threshold, minutes)) -@task() +@task(queue=get_local_queuename) def send_notifications(notification_list, job_id=None): if not isinstance(notification_list, list): raise TypeError("notification_list should be of type list") @@ -336,7 +336,7 @@ def send_notifications(notification_list, job_id=None): logger.exception('Error saving notification {} result.'.format(notification.id)) -@task() +@task(queue=get_local_queuename) def gather_analytics(): from awx.conf.models import Setting from rest_framework.fields import DateTimeField @@ -492,7 +492,7 @@ def awx_isolated_heartbeat(): isolated_manager.IsolatedManager(CallbackQueueDispatcher.dispatch).health_check(isolated_instance_qs) -@task() +@task(queue=get_local_queuename) def awx_periodic_scheduler(): with advisory_lock('awx_periodic_scheduler_lock', wait=False) as acquired: if acquired is False: @@ -549,7 +549,7 @@ def awx_periodic_scheduler(): state.save() -@task() +@task(queue=get_local_queuename) def handle_work_success(task_actual): try: instance = UnifiedJob.get_instance_by_type(task_actual['type'], task_actual['id']) @@ -562,7 +562,7 @@ def handle_work_success(task_actual): schedule_task_manager() -@task() +@task(queue=get_local_queuename) def handle_work_error(task_id, *args, **kwargs): subtasks = kwargs.get('subtasks', None) logger.debug('Executing error task id %s, subtasks: %s' % (task_id, str(subtasks))) @@ -602,7 +602,7 @@ def handle_work_error(task_id, *args, **kwargs): pass -@task() +@task(queue=get_local_queuename) def update_inventory_computed_fields(inventory_id): ''' Signal handler and wrapper around inventory.update_computed_fields to @@ -644,7 +644,7 @@ def update_smart_memberships_for_inventory(smart_inventory): return False -@task() +@task(queue=get_local_queuename) def update_host_smart_inventory_memberships(): smart_inventories = Inventory.objects.filter(kind='smart', host_filter__isnull=False, pending_deletion=False) changed_inventories = set([]) @@ -660,7 +660,7 @@ def update_host_smart_inventory_memberships(): smart_inventory.update_computed_fields() -@task() +@task(queue=get_local_queuename) def delete_inventory(inventory_id, user_id, retries=5): # Delete inventory as user if user_id is None: @@ -1478,7 +1478,7 @@ class BaseTask(object): -@task() +@task(queue=get_local_queuename) class RunJob(BaseTask): ''' Run a job using ansible-playbook. @@ -1911,7 +1911,7 @@ class RunJob(BaseTask): update_inventory_computed_fields.delay(inventory.id) -@task() +@task(queue=get_local_queuename) class RunProjectUpdate(BaseTask): model = ProjectUpdate @@ -2321,7 +2321,7 @@ class RunProjectUpdate(BaseTask): return getattr(settings, 'AWX_PROOT_ENABLED', False) -@task() +@task(queue=get_local_queuename) class RunInventoryUpdate(BaseTask): model = InventoryUpdate @@ -2589,7 +2589,7 @@ class RunInventoryUpdate(BaseTask): ) -@task() +@task(queue=get_local_queuename) class RunAdHocCommand(BaseTask): ''' Run an ad hoc command using ansible. @@ -2779,7 +2779,7 @@ class RunAdHocCommand(BaseTask): isolated_manager_instance.cleanup() -@task() +@task(queue=get_local_queuename) class RunSystemJob(BaseTask): model = SystemJob @@ -2853,7 +2853,7 @@ def _reconstruct_relationships(copy_mapping): new_obj.save() -@task() +@task(queue=get_local_queuename) def deep_copy_model_obj( model_module, model_name, obj_pk, new_obj_pk, user_pk, sub_obj_list, permission_check_func=None diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index f6ad6f6d7c..63a5b74be6 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -423,7 +423,6 @@ BROKER_DURABILITY = True BROKER_POOL_LIMIT = None BROKER_URL = 'redis://localhost:6379' BROKER_TRANSPORT_OPTIONS = {} -CELERY_DEFAULT_QUEUE = 'awx_private_queue' CELERYBEAT_SCHEDULE = { 'tower_scheduler': { 'task': 'awx.main.tasks.awx_periodic_scheduler', @@ -452,14 +451,6 @@ CELERYBEAT_SCHEDULE = { # 'isolated_heartbeat': set up at the end of production.py and development.py } -AWX_CELERY_QUEUES_STATIC = [ - CELERY_DEFAULT_QUEUE, -] - -AWX_CELERY_BCAST_QUEUES_STATIC = [ - 'tower_broadcast_all', -] - ASGI_AMQP = { 'INIT_FUNC': 'awx.prepare_env', 'MODEL': 'awx.main.models.channels.ChannelGroup', From 5818dcc980be31bd521c2670790956491e076154 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Wed, 22 Jan 2020 16:24:08 -0500 Subject: [PATCH 08/41] prefer simple async -> sync * asgiref async_to_sync was causing a Redis connection _for each_ call to emit_channel_notification i.e. every event that the callback receiver processes. This is a "known" issue https://github.com/django/channels_redis/pull/130#issuecomment-424274470 and the advise is to slow downn the rate at which you call async_to_sync. That is not an option for us. Instead, we put the async group_send call onto the event loop for the current thread and wait for it to be processed immediately. The known issue has to do with event loop + socket relationship. Each connection to redis is achieved via a socket. That conection can only be waiting on by the event loop that corresponds to the calling thread. async_to_sync creates a _new thread_ for each invocation. Thus, a new connection to redis is required. Thus, the excess redis connections that can be observed via netstat | grep redis | wc -l. --- awx/main/consumers.py | 59 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/awx/main/consumers.py b/awx/main/consumers.py index bc7aa47089..03173ceffd 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -5,6 +5,7 @@ import logging import codecs import datetime import hmac +import asyncio from django.utils.encoding import force_bytes from django.utils.encoding import smart_str @@ -200,28 +201,66 @@ class EventConsumer(AsyncJsonWebsocketConsumer): await self.send(event['text']) -def emit_channel_notification(group, payload): +def run_sync(func): + event_loop = None try: - payload = json.dumps(payload, cls=DjangoJSONEncoder) + event_loop = asyncio.get_event_loop() + except RuntimeError: + event_loop = asyncio.new_event_loop() + asyncio.set_event_loop(event_loop) + return event_loop.run_until_complete(func) + + +def _dump_payload(payload): + try: + return json.dumps(payload, cls=DjangoJSONEncoder) except ValueError: - logger.error("Invalid payload emitting channel {} on topic: {}".format(group, payload)) + logger.error("Invalid payload to emit") + return None + + +async def emit_channel_notification_async(group, payload): + payload_dumped = _dump_payload(payload) + if payload_dumped is None: + return + + channel_layer = get_channel_layer() + await channel_layer.group_send( + group, + { + "type": "internal.message", + "text": payload_dumped + }, + ) + + await channel_layer.group_send( + BROADCAST_GROUP, + { + "type": "internal.message", + "text": wrap_broadcast_msg(group, payload_dumped), + }, + ) + + +def emit_channel_notification(group, payload): + payload_dumped = _dump_payload(payload) + if payload_dumped is None: return channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( + run_sync(channel_layer.group_send( group, { "type": "internal.message", - "text": payload + "text": payload_dumped }, - ) + )) - async_to_sync(channel_layer.group_send)( + run_sync(channel_layer.group_send( BROADCAST_GROUP, { "type": "internal.message", - "text": wrap_broadcast_msg(group, payload), + "text": wrap_broadcast_msg(group, payload_dumped), }, - ) - + )) From 088373963bb7c2561533e05aafd8bf4e47d28457 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Thu, 23 Jan 2020 16:10:23 -0500 Subject: [PATCH 09/41] satisfy generic Role code * User in channels session is a lazy user class. This does not conform to what the generic Role ancestry code expects. The Role ancestry code expects a User objects. This change converts the lazy object into a proper User object before calling the permission code path. --- awx/main/consumers.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/awx/main/consumers.py b/awx/main/consumers.py index 03173ceffd..aaa4b1f5b1 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -13,6 +13,7 @@ from django.http.cookie import parse_cookie from django.core.serializers.json import DjangoJSONEncoder from django.conf import settings from django.utils.encoding import force_bytes +from django.contrib.auth.models import User from channels.generic.websocket import AsyncJsonWebsocketConsumer from channels.layers import get_channel_layer @@ -142,7 +143,14 @@ class EventConsumer(AsyncJsonWebsocketConsumer): @database_sync_to_async def user_can_see_object_id(self, user_access, oid): - return user_access.get_queryset().filter(pk=oid).exists() + # At this point user is a channels.auth.UserLazyObject object + # This causes problems with our generic role permissions checking. + # Specifically, type(user) != User + # Therefore, get the "real" User objects from the database before + # calling the access permission methods + user_access.user = User.objects.get(id=user_access.user.id) + res = user_access.get_queryset().filter(pk=oid).exists() + return res async def receive_json(self, data): from awx.main.access import consumer_access From feac93fd24c0ebef8fa32370f3a82699db60896b Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 24 Jan 2020 10:53:41 -0500 Subject: [PATCH 10/41] add websocket group unsubscribe reply * This change adds more than just an unsubscribe reply. * Websockets canrequest to join/leave groups. They do so using a single idempotent request. This change replies to group requests over the websockets with the diff of the group subscription. i.e. what groups the user currenntly is in, what groups were left, and what groups were joined. --- awx/main/consumers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/awx/main/consumers.py b/awx/main/consumers.py index aaa4b1f5b1..e11a20bc99 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -204,6 +204,11 @@ class EventConsumer(AsyncJsonWebsocketConsumer): ) logger.debug(f"Channel {self.channel_name} left groups {old_groups} and joined {new_groups_exclusive}") self.scope['session']['groups'] = new_groups + await self.send_json({ + "groups_current": list(new_groups), + "groups_left": list(old_groups), + "groups_joined": list(new_groups_exclusive) + }) async def internal_message(self, event): await self.send(event['text']) From 3f2d757f4ec5a1e881b73da15cfa0d487be04106 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 24 Jan 2020 15:19:50 -0500 Subject: [PATCH 11/41] update awxkit to use new unsubscribe event * Instead of waiting an arbitrary number of seconds. We can now wait the exact amount of time needed to KNOW that we are unsubscribed. This changeset takes advantage of the new subscribe reply semantic. --- awxkit/awxkit/ws.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/awxkit/awxkit/ws.py b/awxkit/awxkit/ws.py index 8005a8ef66..11668641bd 100644 --- a/awxkit/awxkit/ws.py +++ b/awxkit/awxkit/ws.py @@ -4,6 +4,7 @@ import logging import atexit import json import ssl +from datetime import datetime from six.moves.queue import Queue, Empty from six.moves.urllib.parse import urlparse @@ -93,6 +94,7 @@ class WSClient(object): cookie=auth_cookie) self._message_cache = [] self._should_subscribe_to_pending_job = False + self._pending_unsubscribe = threading.Event() def connect(self): wst = threading.Thread(target=self._ws_run_forever, args=(self.ws, {"cert_reqs": ssl.CERT_NONE})) @@ -184,11 +186,17 @@ class WSClient(object): payload['xrftoken'] = self.csrftoken self._send(json.dumps(payload)) - def unsubscribe(self): - self._send(json.dumps(dict(groups={}, xrftoken=self.csrftoken))) - # it takes time for the unsubscribe event to be recieved and consumed and for - # messages to stop being put on the queue for daphne to send to us - time.sleep(5) + def unsubscribe(self, wait=True, timeout=10): + time_start = datetime.now() + if wait: + # Other unnsubscribe events could have caused the edge to trigger. + # This way the _next_ event will trigger our waiting. + self._pending_unsubscribe.clear() + self._send(json.dumps(dict(groups={}, xrftoken=self.csrftoken))) + if not self._pending_unsubscribe.wait(timeout): + raise RuntimeError(f"Failed while waiting on unsubscribe reply because timeout of {timeout} seconds was reached.") + else: + self._send(json.dumps(dict(groups={}, xrftoken=self.csrftoken))) def _on_message(self, message): message = json.loads(message) @@ -202,6 +210,10 @@ class WSClient(object): self._should_subscribe_to_pending_job['events'] == 'project_update_events'): self._update_subscription(message['unified_job_id']) + # unsubscribe acknowledgement + if 'groups_current' in message: + self._pending_unsubscribe.set() + return self._recv_queue.put(message) def _update_subscription(self, job_id): From ea29f4b91ff65b1a28a9484a00e5a3306aa6bfaa Mon Sep 17 00:00:00 2001 From: chris meyers Date: Mon, 27 Jan 2020 15:10:55 -0500 Subject: [PATCH 12/41] account for isolated job status * We can not query the dispatcher running on isolated nodes to see if the playbook is still running because that is the nature of isolated nodes, they don't run the dispatcher nor do they run the message broker. Therefore, we should query the control node that is arbitrating the isolated work. If the control node process in the dispatcher is dead, consider the iso job dead. --- awx/main/models/unified_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 6ab9ee6066..9702340e34 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -1361,7 +1361,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique timeout = 5 try: running = self.celery_task_id in ControlDispatcher( - 'dispatcher', self.execution_node + 'dispatcher', self.controller_node or self.execution_node ).running(timeout=timeout) except socket.timeout: logger.error('could not reach dispatcher on {} within {}s'.format( From 403e9bbfb5cadfec1d92a400798d001611068906 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Wed, 5 Feb 2020 09:03:04 -0500 Subject: [PATCH 13/41] add websocket health information --- awx/main/channels.py | 244 ++++++++++++++++++++++++++++++++++++------ awx/main/consumers.py | 23 ++++ awx/main/routing.py | 1 + 3 files changed, 235 insertions(+), 33 deletions(-) diff --git a/awx/main/channels.py b/awx/main/channels.py index e76b06cf1b..bf950b333f 100644 --- a/awx/main/channels.py +++ b/awx/main/channels.py @@ -4,6 +4,7 @@ import json import logging import aiohttp import asyncio +import datetime from channels_redis.core import RedisChannelLayer from channels.layers import get_channel_layer @@ -17,13 +18,21 @@ from django.core.serializers.json import DjangoJSONEncoder logger = logging.getLogger('awx.main') -def wrap_broadcast_msg(group, message): +def dt_to_seconds(dt): + return int((dt - datetime.datetime(1970,1,1)).total_seconds()) + + +def now_seconds(): + return dt_to_seconds(datetime.datetime.now()) + + +def wrap_broadcast_msg(group, message: str): # TODO: Maybe wrap as "group","message" so that we don't need to # encode/decode as json. return json.dumps(dict(group=group, message=message), cls=DjangoJSONEncoder) -def unwrap_broadcast_msg(payload): +def unwrap_broadcast_msg(payload: dict): return (payload['group'], payload['message']) @@ -36,55 +45,224 @@ def get_broadcast_hosts(): .distinct()] -class RedisGroupBroadcastChannelLayer(RedisChannelLayer): - def __init__(self, *args, **kwargs): - super(RedisGroupBroadcastChannelLayer, self).__init__(*args, **kwargs) +def get_local_host(): + Instance = apps.get_model('main', 'Instance') + return Instance.objects.me().hostname - self.broadcast_hosts = get_broadcast_hosts() - self.broadcast_websockets = set() - loop = asyncio.get_event_loop() - for host in self.broadcast_hosts: - loop.create_task(self.connect(host, settings.BROADCAST_WEBSOCKETS_PORT)) +# Second granularity; Per-minute +class FixedSlidingWindow(): + def __init__(self, start_time=None): + self.buckets = dict() + self.start_time = start_time or now_seconds() - async def connect(self, host, port, secret='abc123', attempt=0): + def cleanup(self, now_bucket=now_seconds()): + if self.start_time + 60 >= now_bucket: + self.start_time = now_bucket - 60 + 1 + + # Delete old entries + for k,v in self.buckets.items(): + if k < self.start_time: + del self.buckets[k] + + def record(self, ts=datetime.datetime.now()): + now_bucket = int((ts-datetime.datetime(1970,1,1)).total_seconds()) + + val = self.buckets.get(now_bucket, 0) + self.buckets[now_bucket] = val + 1 + + self.cleanup(now_bucket) + + def sum(self): + self.cleanup() + return sum(self.buckets.values()) or 0 + + +class Stats(): + def __init__(self, name): + self.name = name + self._messages_received_per_minute = FixedSlidingWindow() + self._messages_received = 0 + self._is_connected = False + self._connection_established_ts = None + + def record_message_received(self, ts=datetime.datetime.now()): + self._messages_received += 1 + self._messages_received_per_minute.record(ts) + + def get_messages_received_total(self): + return self._messages_received + + def get_messages_received_per_minute(self): + self._messages_received_per_minute.sum() + + def record_connection_established(self, ts=datetime.datetime.now()): + self._connection_established_ts = ts + + def record_connection_lost(self, ts=datetime.datetime.now()): + self._connection_established_ts = None + self._is_connected = False + + def get_connection_duration(self): + return (datetime.datetime.now() - self._connection_established_ts).total_seconds() + + + +class WebsocketTask(): + def __init__(self, + name, + event_loop, + remote_host: str, + remote_port: int=settings.BROADCAST_WEBSOCKETS_PORT, + protocol: str=settings.BROADCAST_WEBSOCKETS_PROTOCOL, + verify_ssl: bool=settings.BROADCAST_WEBSOCKETS_VERIFY_CERT, + endpoint: str='broadcast'): + self.name = name + self.stats = Stats(name) + self.event_loop = event_loop + self.remote_host = remote_host + self.remote_port = remote_port + self.endpoint = endpoint + self.protocol = protocol + self.verify_ssl = verify_ssl + self.channel_layer = None + + async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse): + raise RuntimeError("Implement me") + + async def connect(self, attempt): from awx.main.consumers import WebsocketSecretAuthHelper # noqa + logger.debug(f"{self.name} connect attempt {attempt} to {self.remote_host}") + + ''' + Can not put get_channel_layer() in the init code because it is in the init + path of channel layers i.e. RedisChannelLayer() calls our init code. + ''' + if not self.channel_layer: + self.channel_layer = get_channel_layer() if attempt > 0: await asyncio.sleep(5) - channel_layer = get_channel_layer() - uri = f"{settings.BROADCAST_WEBSOCKETS_PROTOCOL}://{host}:{port}/websocket/broadcast/" + uri = f"{self.protocol}://{self.remote_host}:{self.remote_port}/websocket/{self.endpoint}/" timeout = aiohttp.ClientTimeout(total=10) secret_val = WebsocketSecretAuthHelper.construct_secret() try: async with aiohttp.ClientSession(headers={'secret': secret_val}, timeout=timeout) as session: - async with session.ws_connect(uri, ssl=settings.BROADCAST_WEBSOCKETS_VERIFY_CERT) as websocket: - # TODO: Surface a health status of the broadcast interconnect - async for msg in websocket: - if msg.type == aiohttp.WSMsgType.ERROR: - break - elif msg.type == aiohttp.WSMsgType.TEXT: - try: - payload = json.loads(msg.data) - except json.JSONDecodeError: - logmsg = "Failed to decode broadcast message" - if logger.isEnabledFor(logging.DEBUG): - logmsg = "{} {}".format(logmsg, payload) - logger.warn(logmsg) - continue - - (group, message) = unwrap_broadcast_msg(payload) - - await channel_layer.group_send(group, {"type": "internal.message", "text": message}) + async with session.ws_connect(uri, ssl=self.verify_ssl) as websocket: + self.stats.record_connection_established() + attempt = 0 + await self.run_loop(websocket) except Exception as e: # Early on, this is our canary. I'm not sure what exceptions we can really encounter. # Does aiohttp throws an exception if a disconnect happens? logger.warn("Websocket broadcast client exception {}".format(e)) finally: + self.stats.record_connection_lost() # Reconnect - loop = asyncio.get_event_loop() - loop.create_task(self.connect(host, port, secret, attempt=attempt+1)) + self.start(attempt=attempt+1) + + def start(self, attempt=0): + self.event_loop.create_task(self.connect(attempt=attempt)) +class HealthWebsocketTask(WebsocketTask): + def __init__(self, *args, **kwargs): + self.period = kwargs.pop('period', 10) + self.broadcast_stats = kwargs.pop('broadcast_stats', []) + + super().__init__(*args, endpoint='health', **kwargs) + + self.period_abs = None + # Ideally, we send a health beat at exactly the period. In reality + # there is always jitter due to OS needs, system load, etc. + # This variable tracks that offset. + self.last_period_offset = 0 + + async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse): + ''' + now = datetime.datetime.now() + if not self.period_abs: + self.period_abs = now + + sleep_time = self.period_abs + self.period + + + if now <= next_period: + logger.warn("Websocket broadcast missed sending health ping.") + else: + await asyncio.sleep(sleep_time) + + + sleep_time = datetime.datetime.now() - (self.last_period + datetime.timedelta(seconds=PERIOD)) + ''' + + # Start stats loop + self.event_loop.create_task(self.run_calc_stats_loop()) + + # Let this task loop be the send loop + await self.run_send_stats_loop(websocket) + + async def run_calc_stats_loop(self): + """ + Do any periodic calculations needed. i.e. sampling + """ + await asyncio.sleep(1) + + async def run_send_stats_loop(self, websocket: aiohttp.ClientWebSocketResponse): + while True: + msg = { + "sending_host": self.name, + "remote_hosts": [], + } + for s in self.broadcast_stats: + msg['remote_hosts'].append({ + 'name': s.name, + 'messages_received': s.get_messages_received_total(), + 'messages_received_per_minute': s.get_messages_received_per_minute(), + }) + + logger.debug(f"Sending health message {msg}") + await websocket.send_json(msg) + await asyncio.sleep(10) + + +class BroadcastWebsocketTask(WebsocketTask): + async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse): + async for msg in websocket: + self.stats.record_message_received() + + if msg.type == aiohttp.WSMsgType.ERROR: + break + elif msg.type == aiohttp.WSMsgType.TEXT: + try: + payload = json.loads(msg.data) + except json.JSONDecodeError: + logmsg = "Failed to decode broadcast message" + if logger.isEnabledFor(logging.DEBUG): + logmsg = "{} {}".format(logmsg, payload) + logger.warn(logmsg) + continue + + (group, message) = unwrap_broadcast_msg(payload) + + logger.debug(f"{self.name} broadcasting message") + await self.channel_layer.group_send(group, {"type": "internal.message", "text": message}) + + +class RedisGroupBroadcastChannelLayer(RedisChannelLayer): + def __init__(self, *args, **kwargs): + super(RedisGroupBroadcastChannelLayer, self).__init__(*args, **kwargs) + + remote_hosts = get_broadcast_hosts() + loop = asyncio.get_event_loop() + local_hostname = get_local_host() + + broadcast_tasks = [BroadcastWebsocketTask(local_hostname, loop, h) for h in remote_hosts] + broadcast_stats = [t.stats for t in broadcast_tasks] + health_tasks = [HealthWebsocketTask(local_hostname, loop, h, broadcast_stats=broadcast_stats) for h in remote_hosts] + + [t.start() for t in broadcast_tasks] + [t.start() for t in health_tasks] + diff --git a/awx/main/consumers.py b/awx/main/consumers.py index e11a20bc99..e88d9a603d 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -122,6 +122,29 @@ class BroadcastConsumer(AsyncJsonWebsocketConsumer): await self.send(event['text']) +class HealthConsumer(AsyncJsonWebsocketConsumer): + async def connect(self): + try: + WebsocketSecretAuthHelper.is_authorized(self.scope) + except Exception: + await self.close() + return + + # TODO: log ip of connected client + logger.info("Client connected to health endpoint") + await self.accept() + + async def disconnect(self, code): + # TODO: log ip of disconnected client + logger.info("Client disconnected from health endpoint") + + async def receive_json(self, content, **kwargs): + logger.debug(f"Got Health status {content}") + + async def internal_message(self, event): + logger.info("Got internal message from health endpoint .. can this happen?") + + class EventConsumer(AsyncJsonWebsocketConsumer): async def connect(self): user = self.scope['user'] diff --git a/awx/main/routing.py b/awx/main/routing.py index 1efb6159d3..38ba02a890 100644 --- a/awx/main/routing.py +++ b/awx/main/routing.py @@ -7,6 +7,7 @@ from . import consumers websocket_urlpatterns = [ url(r'websocket/$', consumers.EventConsumer), url(r'websocket/broadcast/$', consumers.BroadcastConsumer), + url(r'websocket/health/$', consumers.HealthConsumer), ] application = ProtocolTypeRouter({ From be58906aed1d50eb246d5fd89c7439b8388d446f Mon Sep 17 00:00:00 2001 From: chris meyers Date: Mon, 10 Feb 2020 15:12:11 -0500 Subject: [PATCH 14/41] remove kombu --- awx/main/dispatch/control.py | 1 - awx/main/dispatch/worker/base.py | 9 +++---- .../commands/run_callback_receiver.py | 27 +++++++++---------- .../management/commands/run_dispatcher.py | 4 +-- awx/main/queue.py | 2 -- .../tests/functional/api/test_settings.py | 17 ++++++------ awx/settings/defaults.py | 4 --- 7 files changed, 25 insertions(+), 39 deletions(-) diff --git a/awx/main/dispatch/control.py b/awx/main/dispatch/control.py index a584c9dfe5..684cdae806 100644 --- a/awx/main/dispatch/control.py +++ b/awx/main/dispatch/control.py @@ -5,7 +5,6 @@ import random import json from awx.main.dispatch import get_local_queuename -from kombu import Queue, Exchange, Producer, Consumer, Connection from . import pg_bus_conn diff --git a/awx/main/dispatch/worker/base.py b/awx/main/dispatch/worker/base.py index b90b41ce1c..674892a7b5 100644 --- a/awx/main/dispatch/worker/base.py +++ b/awx/main/dispatch/worker/base.py @@ -13,8 +13,6 @@ from uuid import UUID from queue import Empty as QueueEmpty from django import db -from kombu import Producer -from kombu.mixins import ConsumerMixin from django.conf import settings from awx.main.dispatch.pool import WorkerPool @@ -44,11 +42,10 @@ class WorkerSignalHandler: class AWXConsumerBase(object): - def __init__(self, name, connection, worker, queues=[], pool=None): + def __init__(self, name, worker, queues=[], pool=None): self.should_stop = False self.name = name - self.connection = connection self.total_messages = 0 self.queues = queues self.worker = worker @@ -110,7 +107,7 @@ class AWXConsumerBase(object): # Child should implement other things here def stop(self, signum, frame): - self.should_stop = True # this makes the kombu mixin stop consuming + self.should_stop = True logger.warn('received {}, stopping'.format(signame(signum))) self.worker.on_stop() raise SystemExit() @@ -142,6 +139,8 @@ class AWXConsumerPG(AWXConsumerBase): conn.listen(queue) for e in conn.events(): self.process_task(json.loads(e.payload)) + if self.should_stop: + return except psycopg2.InterfaceError: logger.warn("Stale Postgres message bus connection, reconnecting") continue diff --git a/awx/main/management/commands/run_callback_receiver.py b/awx/main/management/commands/run_callback_receiver.py index f1c1aed0c3..7e28330067 100644 --- a/awx/main/management/commands/run_callback_receiver.py +++ b/awx/main/management/commands/run_callback_receiver.py @@ -3,7 +3,6 @@ from django.conf import settings from django.core.management.base import BaseCommand -from kombu import Exchange, Queue, Connection from awx.main.dispatch.worker import AWXConsumerRedis, CallbackBrokerWorker @@ -17,17 +16,15 @@ class Command(BaseCommand): help = 'Launch the job callback receiver' def handle(self, *arg, **options): - with Connection(settings.BROKER_URL, transport_options=settings.BROKER_TRANSPORT_OPTIONS) as conn: - consumer = None - try: - consumer = AWXConsumerRedis( - 'callback_receiver', - conn, - CallbackBrokerWorker(), - queues=[getattr(settings, 'CALLBACK_QUEUE', '')], - ) - consumer.run() - except KeyboardInterrupt: - print('Terminating Callback Receiver') - if consumer: - consumer.stop() + consumer = None + try: + consumer = AWXConsumerRedis( + 'callback_receiver', + CallbackBrokerWorker(), + queues=[getattr(settings, 'CALLBACK_QUEUE', '')], + ) + consumer.run() + except KeyboardInterrupt: + print('Terminating Callback Receiver') + if consumer: + consumer.stop() diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py index b678797026..d12b23f275 100644 --- a/awx/main/management/commands/run_dispatcher.py +++ b/awx/main/management/commands/run_dispatcher.py @@ -5,8 +5,7 @@ import logging from django.conf import settings from django.core.cache import cache as django_cache from django.core.management.base import BaseCommand -from django.db import connection as django_connection -from kombu import Exchange, Queue +from django.db import connection as django_connection, connections from awx.main.utils.handlers import AWXProxyHandler from awx.main.dispatch import get_local_queuename, reaper @@ -66,7 +65,6 @@ class Command(BaseCommand): queues = ['tower_broadcast_all', get_local_queuename()] consumer = AWXConsumerPG( 'dispatcher', - None, TaskWorker(), queues, AutoscalePool(min_workers=4) diff --git a/awx/main/queue.py b/awx/main/queue.py index 8d4fffdf87..38bea6fc2c 100644 --- a/awx/main/queue.py +++ b/awx/main/queue.py @@ -10,8 +10,6 @@ import redis # Django from django.conf import settings -# Kombu -from kombu import Exchange, Producer, Connection __all__ = ['CallbackQueueDispatcher'] diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py index a88aa8c20b..723b942c94 100644 --- a/awx/main/tests/functional/api/test_settings.py +++ b/awx/main/tests/functional/api/test_settings.py @@ -8,7 +8,7 @@ import os import time from django.conf import settings -from kombu.utils.url import parse_url +import redis # Mock from unittest import mock @@ -390,11 +390,10 @@ def test_saml_x509cert_validation(patch, get, admin, headers): @pytest.mark.django_db def test_broker_url_with_special_characters(): - settings.BROKER_URL = 'amqp://guest:a@ns:ibl3#@rabbitmq:5672//' - url = parse_url(settings.BROKER_URL) - assert url['transport'] == 'amqp' - assert url['hostname'] == 'rabbitmq' - assert url['port'] == 5672 - assert url['userid'] == 'guest' - assert url['password'] == 'a@ns:ibl3#' - assert url['virtual_host'] == '/' + settings.BROKER_URL = 'redis://unused:a@ns:ibl3#@redis-fancy:5672/?db=mydb' + cli = redis.from_url(settings.BROKER_URL) + assert cli.host == 'redis-fancy' + assert cli.port == 5672 + # Note: There are no usernames in redis + assert cli.password == 'a@ns:ibl3#' + assert cli.db == 'mydb' diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 63a5b74be6..798f8ba131 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1112,10 +1112,6 @@ LOGGING = { 'handlers': ['console', 'file', 'tower_warnings'], 'level': 'WARNING', }, - 'kombu': { - 'handlers': ['console', 'file', 'tower_warnings'], - 'level': 'WARNING', - }, 'rest_framework.request': { 'handlers': ['console', 'file', 'tower_warnings'], 'level': 'WARNING', From e94bb44082411b17fd9fb40b54182db37323c1f2 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 11 Feb 2020 09:10:34 -0500 Subject: [PATCH 15/41] replace rabbitmq with redis * local awx docker-compose and image build only. --- installer/inventory | 4 ---- installer/roles/image_build/files/launch_awx.sh | 2 +- installer/roles/image_build/files/settings.py | 11 ++++------- .../roles/image_build/files/supervisor_task.conf | 13 +------------ .../image_build/templates/launch_awx_task.sh.j2 | 2 +- installer/roles/local_docker/defaults/main.yml | 12 +++--------- .../roles/local_docker/templates/credentials.py.j2 | 14 +++++--------- .../local_docker/templates/docker-compose.yml.j2 | 14 +++++--------- .../roles/local_docker/templates/environment.sh.j2 | 4 ++-- 9 files changed, 22 insertions(+), 54 deletions(-) diff --git a/installer/inventory b/installer/inventory index aa7f588bea..b627a00b69 100644 --- a/installer/inventory +++ b/installer/inventory @@ -93,10 +93,6 @@ pg_port=5432 # containerized postgres deployment on OpenShift # pg_admin_password=postgrespass -# RabbitMQ Configuration -rabbitmq_password=awxpass -rabbitmq_erlang_cookie=cookiemonster - # Use a local distribution build container image for building the AWX package # This is helpful if you don't want to bother installing the build-time dependencies as # it is taken care of already. diff --git a/installer/roles/image_build/files/launch_awx.sh b/installer/roles/image_build/files/launch_awx.sh index 97909bc29b..360be51528 100755 --- a/installer/roles/image_build/files/launch_awx.sh +++ b/installer/roles/image_build/files/launch_awx.sh @@ -9,7 +9,7 @@ source /etc/tower/conf.d/environment.sh ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$MEMCACHED_HOST port=$MEMCACHED_PORT" all -ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$RABBITMQ_HOST port=$RABBITMQ_PORT" all +ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$REDIS_HOST port=$REDIS_PORT" all ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m postgresql_db --become-user $DATABASE_USER -a "name=$DATABASE_NAME owner=$DATABASE_USER login_user=$DATABASE_USER login_host=$DATABASE_HOST login_password=$DATABASE_PASSWORD port=$DATABASE_PORT" all awx-manage collectstatic --noinput --clear diff --git a/installer/roles/image_build/files/settings.py b/installer/roles/image_build/files/settings.py index ce4e073949..6fe4306ec0 100644 --- a/installer/roles/image_build/files/settings.py +++ b/installer/roles/image_build/files/settings.py @@ -85,17 +85,14 @@ DATABASES = { if os.getenv("DATABASE_SSLMODE", False): DATABASES['default']['OPTIONS'] = {'sslmode': os.getenv("DATABASE_SSLMODE")} -BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format( - os.getenv("RABBITMQ_USER", None), - os.getenv("RABBITMQ_PASSWORD", None), - os.getenv("RABBITMQ_HOST", None), - os.getenv("RABBITMQ_PORT", "5672"), - os.getenv("RABBITMQ_VHOST", "tower")) +BROKER_URL = 'redis://{}:{}'.format( + os.getenv("REDIS_HOST", None), + os.getenv("REDIS_PORT", "6379"), CHANNEL_LAYERS = { 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer', 'ROUTING': 'awx.main.routing.channel_routing', - 'CONFIG': {'url': BROKER_URL}} + 'CONFIG': {'hosts': [(os.getenv("REDIS_HOST", None), int(os.getenv("REDIS_PORT", 6379)))]}} } USE_X_FORWARDED_PORT = True diff --git a/installer/roles/image_build/files/supervisor_task.conf b/installer/roles/image_build/files/supervisor_task.conf index e7e94196e6..a0100980b2 100644 --- a/installer/roles/image_build/files/supervisor_task.conf +++ b/installer/roles/image_build/files/supervisor_task.conf @@ -26,19 +26,8 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -[program:channels-worker] -command = awx-manage runworker --only-channels websocket.* -directory = /var/lib/awx -autostart = true -autorestart = true -stopwaitsecs = 5 -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 - [group:tower-processes] -programs=dispatcher,callback-receiver,channels-worker +programs=dispatcher,callback-receiver priority=5 # TODO: Exit Handler diff --git a/installer/roles/image_build/templates/launch_awx_task.sh.j2 b/installer/roles/image_build/templates/launch_awx_task.sh.j2 index c908ad3f62..532f380c62 100755 --- a/installer/roles/image_build/templates/launch_awx_task.sh.j2 +++ b/installer/roles/image_build/templates/launch_awx_task.sh.j2 @@ -9,7 +9,7 @@ source /etc/tower/conf.d/environment.sh ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$MEMCACHED_HOST port=$MEMCACHED_PORT" all -ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$RABBITMQ_HOST port=$RABBITMQ_PORT" all +ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$REDIS_HOST port=$REDIS_PORT" all ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m postgresql_db --become-user $DATABASE_USER -a "name=$DATABASE_NAME owner=$DATABASE_USER login_user=$DATABASE_USER login_host=$DATABASE_HOST login_password=$DATABASE_PASSWORD port=$DATABASE_PORT" all if [ -z "$AWX_SKIP_MIGRATIONS" ]; then diff --git a/installer/roles/local_docker/defaults/main.yml b/installer/roles/local_docker/defaults/main.yml index 22f74d47ee..9a9277ae68 100644 --- a/installer/roles/local_docker/defaults/main.yml +++ b/installer/roles/local_docker/defaults/main.yml @@ -1,19 +1,13 @@ --- dockerhub_version: "{{ lookup('file', playbook_dir + '/../VERSION') }}" -rabbitmq_version: "3.7.4" -rabbitmq_image: "ansible/awx_rabbitmq:{{rabbitmq_version}}" -rabbitmq_default_vhost: "awx" -rabbitmq_erlang_cookie: "cookiemonster" -rabbitmq_hostname: "rabbitmq" -rabbitmq_port: "5672" -rabbitmq_user: "guest" -rabbitmq_password: "guest" +redis_image: "redis" +redis_hostname: "redis" +redis_port: "6379" postgresql_version: "10" postgresql_image: "postgres:{{postgresql_version}}" - memcached_image: "memcached" memcached_version: "alpine" memcached_hostname: "memcached" diff --git a/installer/roles/local_docker/templates/credentials.py.j2 b/installer/roles/local_docker/templates/credentials.py.j2 index 73951ca803..be71a5dc4a 100644 --- a/installer/roles/local_docker/templates/credentials.py.j2 +++ b/installer/roles/local_docker/templates/credentials.py.j2 @@ -10,17 +10,13 @@ DATABASES = { } } -BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format( - "{{ rabbitmq_user }}", - "{{ rabbitmq_password }}", - "{{ rabbitmq_hostname | default('rabbitmq')}}", - "{{ rabbitmq_port }}", - "{{ rabbitmq_default_vhost }}") +BROKER_URL = 'redis://{}:{}/'.format( + "{{ redis_hostname }}", + "{{ redis_port }}",) CHANNEL_LAYERS = { - 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer', - 'ROUTING': 'awx.main.routing.channel_routing', - 'CONFIG': {'url': BROKER_URL}} + 'default': {'BACKEND': 'awx.main.channels.RedisGroupBroadcastChannelLayer', + 'CONFIG': {'hosts': [("{{ redis_hostname }}", {{ redis_port|int }})]}} } CACHES = { diff --git a/installer/roles/local_docker/templates/docker-compose.yml.j2 b/installer/roles/local_docker/templates/docker-compose.yml.j2 index cf799e3c09..2fd82a1005 100644 --- a/installer/roles/local_docker/templates/docker-compose.yml.j2 +++ b/installer/roles/local_docker/templates/docker-compose.yml.j2 @@ -6,7 +6,7 @@ services: image: {{ awx_web_docker_actual_image }} container_name: awx_web depends_on: - - rabbitmq + - redis - memcached {% if pg_hostname is not defined %} - postgres @@ -63,7 +63,7 @@ services: image: {{ awx_task_docker_actual_image }} container_name: awx_task depends_on: - - rabbitmq + - redis - memcached - web {% if pg_hostname is not defined %} @@ -111,15 +111,11 @@ services: https_proxy: {{ https_proxy | default('') }} no_proxy: {{ no_proxy | default('') }} - rabbitmq: - image: {{ rabbitmq_image }} - container_name: awx_rabbitmq + redis: + image: {{ redis_image }} + container_name: awx_redis restart: unless-stopped environment: - RABBITMQ_DEFAULT_VHOST: "{{ rabbitmq_default_vhost }}" - RABBITMQ_DEFAULT_USER: "{{ rabbitmq_user }}" - RABBITMQ_DEFAULT_PASS: "{{ rabbitmq_password | quote }}" - RABBITMQ_ERLANG_COOKIE: {{ rabbitmq_erlang_cookie }} http_proxy: {{ http_proxy | default('') }} https_proxy: {{ https_proxy | default('') }} no_proxy: {{ no_proxy | default('') }} diff --git a/installer/roles/local_docker/templates/environment.sh.j2 b/installer/roles/local_docker/templates/environment.sh.j2 index 817c270e11..83b584b261 100644 --- a/installer/roles/local_docker/templates/environment.sh.j2 +++ b/installer/roles/local_docker/templates/environment.sh.j2 @@ -8,7 +8,7 @@ DATABASE_ADMIN_PASSWORD={{ pg_admin_password|quote }} {% endif %} MEMCACHED_HOST={{ memcached_hostname|default('memcached') }} MEMCACHED_PORT={{ memcached_port|default('11211')|quote }} -RABBITMQ_HOST={{ rabbitmq_hostname|default('rabbitmq')|quote }} -RABBITMQ_PORT={{ rabbitmq_port|default('5672')|quote }} +REDIS_HOST={{ redis_hostname|quote }} +REDIS_PORT={{ redis_port|quote }} AWX_ADMIN_USER={{ admin_user|quote }} AWX_ADMIN_PASSWORD={{ admin_password|quote }} From 45ce6d794eb6a5f9896edfda09573a7d47861be2 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 13 Feb 2020 13:16:25 -0500 Subject: [PATCH 16/41] Initial migration of rabbitmq -> redis for k8s installs --- awx/main/channels.py | 11 +- awx/main/managers.py | 19 +- .../0109_v370_instance_ip_address.py | 18 ++ awx/main/models/ha.py | 7 + installer/inventory | 6 +- installer/roles/kubernetes/defaults/main.yml | 17 +- installer/roles/kubernetes/tasks/main.yml | 4 - .../roles/kubernetes/tasks/ssl_cert_gen.yml | 60 ---- .../kubernetes/templates/configmap.yml.j2 | 2 + .../kubernetes/templates/credentials.py.j2 | 18 +- .../kubernetes/templates/deployment.yml.j2 | 280 ++---------------- .../kubernetes/templates/environment.sh.j2 | 2 - .../templates/management-pod.yml.j2 | 1 + .../roles/kubernetes/templates/secret.yml.j2 | 17 -- 14 files changed, 84 insertions(+), 378 deletions(-) create mode 100644 awx/main/migrations/0109_v370_instance_ip_address.py delete mode 100644 installer/roles/kubernetes/tasks/ssl_cert_gen.yml diff --git a/awx/main/channels.py b/awx/main/channels.py index bf950b333f..5673166bb9 100644 --- a/awx/main/channels.py +++ b/awx/main/channels.py @@ -38,11 +38,12 @@ def unwrap_broadcast_msg(payload: dict): def get_broadcast_hosts(): Instance = apps.get_model('main', 'Instance') - return [h[0] for h in Instance.objects.filter(rampart_groups__controller__isnull=True) - .exclude(hostname=Instance.objects.me().hostname) - .order_by('hostname') - .values_list('hostname') - .distinct()] + instances = Instance.objects.filter(rampart_groups__controller__isnull=True) \ + .exclude(hostname=Instance.objects.me().hostname) \ + .order_by('hostname') \ + .values('hostname', 'ip_address') \ + .distinct() + return [i['ip_address'] or i['hostname'] for i in instances] def get_local_host(): diff --git a/awx/main/managers.py b/awx/main/managers.py index 610273119b..ea65b36234 100644 --- a/awx/main/managers.py +++ b/awx/main/managers.py @@ -3,6 +3,7 @@ import sys import logging +import os from django.db import models from django.conf import settings @@ -114,7 +115,7 @@ class InstanceManager(models.Manager): return node[0] raise RuntimeError("No instance found with the current cluster host id") - def register(self, uuid=None, hostname=None): + def register(self, uuid=None, hostname=None, ip_address=None): if not uuid: uuid = settings.SYSTEM_UUID if not hostname: @@ -122,13 +123,23 @@ class InstanceManager(models.Manager): with advisory_lock('instance_registration_%s' % hostname): instance = self.filter(hostname=hostname) if instance.exists(): - return (False, instance[0]) - instance = self.create(uuid=uuid, hostname=hostname, capacity=0) + instance = instance.get() + if instance.ip_address != ip_address: + instance.ip_address = ip_address + instance.save() + return (True, instance) + else: + return (False, instance) + instance = self.create(uuid=uuid, + hostname=hostname, + ip_address=ip_address, + capacity=0) return (True, instance) def get_or_register(self): if settings.AWX_AUTO_DEPROVISION_INSTANCES: - return self.register() + pod_ip = os.environ.get('MY_POD_IP') + return self.register(ip_address=pod_ip) else: return (False, self.me()) diff --git a/awx/main/migrations/0109_v370_instance_ip_address.py b/awx/main/migrations/0109_v370_instance_ip_address.py new file mode 100644 index 0000000000..f3c97d5cff --- /dev/null +++ b/awx/main/migrations/0109_v370_instance_ip_address.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.8 on 2020-02-12 17:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0108_v370_unifiedjob_dependencies_processed'), + ] + + operations = [ + migrations.AddField( + model_name='instance', + name='ip_address', + field=models.CharField(blank=True, default=None, max_length=50, null=True, unique=True), + ), + ] diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py index acc529a29a..ac7df97e10 100644 --- a/awx/main/models/ha.py +++ b/awx/main/models/ha.py @@ -53,6 +53,13 @@ class Instance(HasPolicyEditsMixin, BaseModel): uuid = models.CharField(max_length=40) hostname = models.CharField(max_length=250, unique=True) + ip_address = models.CharField( + blank=True, + null=True, + default=None, + max_length=50, + unique=True, + ) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) last_isolated_check = models.DateTimeField( diff --git a/installer/inventory b/installer/inventory index b627a00b69..3b325d543b 100644 --- a/installer/inventory +++ b/installer/inventory @@ -38,7 +38,7 @@ dockerhub_base=ansible # kubernetes_ingress_tls_secret=awx-cert # Kubernetes and Openshift Install Resource Requests -# These are the request and limit values for a pod's container for task/web/rabbitmq/memcached/management. +# These are the request and limit values for a pod's container for task/web/redis/memcached/management. # The total amount of requested resources for a pod is the sum of all # resources requested by all containers in the pod # A cpu_request of 1500 is 1.5 cores for the container to start out with. @@ -52,8 +52,8 @@ dockerhub_base=ansible # task_mem_limit=4 # web_cpu_limit=1000 # web_mem_limit=2 -# rabbitmq_cpu_limit=1000 -# rabbitmq_mem_limit=3 +# redis_cpu_limit=1000 +# redis_mem_limit=3 # memcached_cpu_limit=1000 # memcached_mem_limit=2 # management_cpu_limit=2000 diff --git a/installer/roles/kubernetes/defaults/main.yml b/installer/roles/kubernetes/defaults/main.yml index 4dc863ea9e..86680f6fa0 100644 --- a/installer/roles/kubernetes/defaults/main.yml +++ b/installer/roles/kubernetes/defaults/main.yml @@ -6,10 +6,6 @@ admin_user: 'admin' admin_email: 'root@localhost' admin_password: '' -rabbitmq_user: 'awx' -rabbitmq_password: '' -rabbitmq_erlang_cookie: '' - kubernetes_base_path: "{{ local_base_config_path|default('/tmp') }}/{{ kubernetes_deployment_name }}-config" kubernetes_task_version: "{{ tower_package_version | default(dockerhub_version) }}" @@ -24,16 +20,18 @@ web_cpu_request: 500 task_mem_request: 2 task_cpu_request: 1500 -rabbitmq_mem_request: 2 -rabbitmq_cpu_request: 500 +redis_mem_request: 2 +redis_cpu_request: 500 + +kubernetes_redis_image: "redis" +kubernetes_redis_image_tag: "latest" +kubernetes_redis_hostname: "localhost" +kubernetes_redis_port: "6379" memcached_hostname: localhost memcached_mem_request: 1 memcached_cpu_request: 500 -kubernetes_rabbitmq_version: "3.7.21" -kubernetes_rabbitmq_image: "ansible/awx_rabbitmq" - kubernetes_memcached_version: "latest" kubernetes_memcached_image: "memcached" @@ -56,6 +54,5 @@ custom_venvs_path: "/opt/custom-venvs" custom_venvs_python: "python2" ca_trust_bundle: "/etc/pki/tls/certs/ca-bundle.crt" -rabbitmq_use_ssl: false container_groups_image: "ansible/ansible-runner" diff --git a/installer/roles/kubernetes/tasks/main.yml b/installer/roles/kubernetes/tasks/main.yml index d437d5f890..dc18eefe1c 100644 --- a/installer/roles/kubernetes/tasks/main.yml +++ b/installer/roles/kubernetes/tasks/main.yml @@ -194,10 +194,6 @@ when: kubernetes_web_image is not defined when: docker_registry is defined -- name: Generate SSL certificates for RabbitMQ, if needed - include_tasks: ssl_cert_gen.yml - when: "rabbitmq_use_ssl|default(False)|bool" - - name: Determine StatefulSet api version set_fact: kubernetes_statefulset_api_version: "{{ 'apps/v1' if kube_api_version is version('1.9', '>=') else 'apps/v1beta1' }}" diff --git a/installer/roles/kubernetes/tasks/ssl_cert_gen.yml b/installer/roles/kubernetes/tasks/ssl_cert_gen.yml deleted file mode 100644 index c1b7199028..0000000000 --- a/installer/roles/kubernetes/tasks/ssl_cert_gen.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- - -- name: Create temporary directory - tempfile: - state: directory - prefix: "tower-install-rmq-certs" - register: rmq_cert_tempdir - notify: remove-rmq_cert_tempdir - -- name: Generate CA private key - openssl_privatekey: - path: '{{ rmq_cert_tempdir.path }}/ca.key' - mode: "0600" - -- name: Generate CA CSR - openssl_csr: - path: '{{ rmq_cert_tempdir.path }}/ca.csr' - privatekey_path: '{{ rmq_cert_tempdir.path }}/ca.key' - common_name: 'rabbitmq-ca' - basic_constraints: 'CA:TRUE' - mode: "0600" - -- name: Generate CA certificate - openssl_certificate: - path: '{{ rmq_cert_tempdir.path }}/ca.crt' - csr_path: '{{ rmq_cert_tempdir.path }}/ca.csr' - privatekey_path: '{{ rmq_cert_tempdir.path }}/ca.key' - provider: selfsigned - selfsigned_not_after: "+36524d" - mode: "0600" - -- name: Generate server private key - openssl_privatekey: - path: '{{ rmq_cert_tempdir.path }}/server.key' - mode: "0600" - -- name: Generate server CSR - openssl_csr: - path: '{{ rmq_cert_tempdir.path }}/server.csr' - privatekey_path: '{{ rmq_cert_tempdir.path }}/server.key' - common_name: 'rabbitmq-server' - mode: "0600" - -- name: Generate server certificate - openssl_certificate: - path: "{{ rmq_cert_tempdir.path }}/server.crt" - csr_path: "{{ rmq_cert_tempdir.path }}/server.csr" - privatekey_path: "{{ rmq_cert_tempdir.path }}/server.key" - provider: ownca - ownca_path: "{{ rmq_cert_tempdir.path }}/ca.crt" - ownca_privatekey_path: "{{ rmq_cert_tempdir.path }}/ca.key" - ownca_not_after: "+36500d" - mode: "0600" - -- name: Create combined certificate - assemble: - src: "{{ rmq_cert_tempdir.path }}" - regexp: "server.crt|server.key" - dest: "{{ rmq_cert_tempdir.path }}/server-combined.pem" - mode: "0600" diff --git a/installer/roles/kubernetes/templates/configmap.yml.j2 b/installer/roles/kubernetes/templates/configmap.yml.j2 index 479e859b48..9b904e709d 100644 --- a/installer/roles/kubernetes/templates/configmap.yml.j2 +++ b/installer/roles/kubernetes/templates/configmap.yml.j2 @@ -205,3 +205,5 @@ data: USE_X_FORWARDED_PORT = True AWX_CONTAINER_GROUP_DEFAULT_IMAGE = "{{ container_groups_image }}" + BROADCAST_WEBSOCKETS_PORT = 8052 + BROADCAST_WEBSOCKETS_PROTOCOL = 'http' diff --git a/installer/roles/kubernetes/templates/credentials.py.j2 b/installer/roles/kubernetes/templates/credentials.py.j2 index f353796bb1..fd68abc523 100644 --- a/installer/roles/kubernetes/templates/credentials.py.j2 +++ b/installer/roles/kubernetes/templates/credentials.py.j2 @@ -12,14 +12,12 @@ DATABASES = { }, } } -BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format( - "{{ rabbitmq_user }}", - "{{ rabbitmq_password }}", - "localhost", - "5672", - "awx") + +BROKER_URL = 'redis://{}:{}/'.format( + "{{ kubernetes_redis_hostname }}", + "{{ kubernetes_redis_port }}",) + CHANNEL_LAYERS = { - 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer', - 'ROUTING': 'awx.main.routing.channel_routing', - 'CONFIG': {'url': BROKER_URL}} -} + 'default': {'BACKEND': 'awx.main.channels.RedisGroupBroadcastChannelLayer', + 'CONFIG': {'hosts': [("{{ kubernetes_redis_hostname }}", {{ kubernetes_redis_port|int }})]}} +} \ No newline at end of file diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index 2e1103e691..ec4c0a65a4 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -15,131 +15,6 @@ imagePullSecrets: - name: "{{ kubernetes_image_pull_secrets }}" {% endif %} ---- -kind: Service -apiVersion: v1 -metadata: - namespace: {{ kubernetes_namespace }} - name: rabbitmq - labels: - app: {{ kubernetes_deployment_name }} - type: LoadBalancer -spec: - type: NodePort - ports: - - name: http - protocol: TCP - port: 15672 - targetPort: 15672 - - name: amqp - protocol: TCP - port: 5672 - targetPort: 5672 - selector: - app: {{ kubernetes_deployment_name }} - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: rabbitmq-config - namespace: {{ kubernetes_namespace }} -data: - enabled_plugins: | - [rabbitmq_management,rabbitmq_peer_discovery_k8s]. - rabbitmq_definitions.json: | - { - "users":[{"name": "{{ rabbitmq_user }}", "password": "{{ rabbitmq_password }}", "tags": "administrator"}], - "permissions":[ - {"user":"{{ rabbitmq_user }}","vhost":"awx","configure":".*","write":".*","read":".*"} - ], - "vhosts":[{"name":"awx"}], - "policies":[ - {"vhost":"awx","name":"ha-all","pattern":".*","definition":{"ha-mode":"all","ha-sync-mode":"automatic"}} - ] - } - rabbitmq.conf: | - ## Clustering - management.load_definitions = /etc/rabbitmq/rabbitmq_definitions.json - cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s - cluster_formation.k8s.host = kubernetes.default.svc - cluster_formation.k8s.address_type = ip - cluster_formation.node_cleanup.interval = 10 - cluster_formation.node_cleanup.only_log_warning = false - cluster_partition_handling = autoheal - ## queue master locator - queue_master_locator=min-masters - ## enable guest user - loopback_users.guest = false -{% if rabbitmq_use_ssl|default(False)|bool %} - ssl_options.cacertfile=/etc/pki/rabbitmq/ca.crt - ssl_options.certfile=/etc/pki/rabbitmq/server-combined.pem - ssl_options.verify=verify_peer -{% endif %} - rabbitmq-env.conf: | - NODENAME=${RABBITMQ_NODENAME} - USE_LONGNAME=true -{% if rabbitmq_use_ssl|default(False)|bool %} - ERL_SSL_PATH=$(erl -eval 'io:format("~p", [code:lib_dir(ssl, ebin)]),halt().' -noshell) - SSL_ADDITIONAL_ERL_ARGS="-pa '$ERL_SSL_PATH' -proto_dist inet_tls -ssl_dist_opt server_certfile /etc/pki/rabbitmq/server-combined.pem -ssl_dist_opt server_secure_renegotiate true client_secure_renegotiate true" - SERVER_ADDITIONAL_ERL_ARGS="$SERVER_ADDITIONAL_ERL_ARGS $SSL_ADDITIONAL_ERL_ARGS" - CTL_ERL_ARGS="$SSL_ADDITIONAL_ERL_ARGS" -{% endif %} - -{% if kubernetes_context is defined %} ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: endpoint-reader - namespace: {{ kubernetes_namespace }} -rules: -- apiGroups: [""] - resources: ["endpoints"] - verbs: ["get"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: endpoint-reader - namespace: {{ kubernetes_namespace }} -subjects: -- kind: ServiceAccount - name: awx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: endpoint-reader -{% endif %} - -{% if openshift_host is defined %} ---- -kind: Role -apiVersion: v1 -metadata: - name: endpoint-reader - namespace: {{ kubernetes_namespace }} -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get"] ---- -kind: RoleBinding -apiVersion: v1 -metadata: - name: endpoint-reader - namespace: {{ kubernetes_namespace }} -roleRef: - name: endpoint-reader - namespace: {{ kubernetes_namespace }} -subjects: - - kind: ServiceAccount - name: awx - namespace: {{ kubernetes_namespace }} -userNames: - - system:serviceaccount:{{ kubernetes_namespace }}:awx -{% endif %} - --- apiVersion: {{ kubernetes_statefulset_api_version }} kind: StatefulSet @@ -165,7 +40,6 @@ spec: service: django app: {{ kubernetes_deployment_name }} spec: - serviceAccountName: awx terminationGracePeriodSeconds: 10 {% if custom_venvs is defined %} {% set trusted_hosts = "" %} @@ -266,7 +140,7 @@ spec: {% if web_cpu_limit is defined %} cpu: "{{ web_cpu_limit }}m" {% endif %} - - name: {{ kubernetes_deployment_name }}-celery + - name: {{ kubernetes_deployment_name }}-task securityContext: privileged: true image: "{{ kubernetes_task_image }}:{{ kubernetes_task_version }}" @@ -303,6 +177,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.uid + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP resources: requests: memory: "{{ task_mem_request }}Gi" @@ -316,72 +194,25 @@ spec: {% if task_cpu_limit is defined %} cpu: "{{ task_cpu_limit }}m" {% endif %} - - name: {{ kubernetes_deployment_name }}-rabbit - image: "{{ kubernetes_rabbitmq_image }}:{{ kubernetes_rabbitmq_version }}" + - name: {{ kubernetes_deployment_name }}-redis + image: {{ kubernetes_redis_image }}:{{ kubernetes_redis_image_tag }} imagePullPolicy: Always ports: - - name: http + - name: redis protocol: TCP - containerPort: 15672 - - name: amqp - protocol: TCP - containerPort: 5672 - livenessProbe: - exec: - command: - - /usr/local/bin/healthchecks/rabbit_health_node.py - initialDelaySeconds: 30 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - - /usr/local/bin/healthchecks/rabbit_health_node.py - initialDelaySeconds: 10 - timeoutSeconds: 10 - env: - - name: MY_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: RABBITMQ_USE_LONGNAME - value: "true" - - name: RABBITMQ_NODENAME - value: "rabbit@$(MY_POD_IP)" - - name: RABBITMQ_ERLANG_COOKIE - valueFrom: - secretKeyRef: - name: "{{ kubernetes_deployment_name }}-secrets" - key: rabbitmq_erlang_cookie - - name: K8S_SERVICE_NAME - value: "rabbitmq" - - name: RABBITMQ_USER - value: {{ rabbitmq_user }} - - name: RABBITMQ_PASSWORD - valueFrom: - secretKeyRef: - name: "{{ kubernetes_deployment_name }}-secrets" - key: rabbitmq_password - volumeMounts: - - name: rabbitmq-config - mountPath: /etc/rabbitmq - - name: rabbitmq-healthchecks - mountPath: /usr/local/bin/healthchecks -{% if rabbitmq_use_ssl|default(False)|bool %} - - name: "{{ kubernetes_deployment_name }}-rabbitmq-certs-vol" - mountPath: /etc/pki/rabbitmq -{% endif %} + containerPort: 6379 resources: requests: - memory: "{{ rabbitmq_mem_request }}Gi" - cpu: "{{ rabbitmq_cpu_request }}m" -{% if rabbitmq_mem_limit is defined or rabbitmq_cpu_limit is defined %} + memory: "{{ redis_mem_request }}Gi" + cpu: "{{ redis_cpu_request }}m" +{% if redis_mem_limit is defined or redis_cpu_limit is defined %} limits: {% endif %} -{% if rabbitmq_mem_limit is defined %} - memory: "{{ rabbitmq_mem_limit }}Gi" +{% if redis_mem_limit is defined %} + memory: "{{ redis_mem_limit }}Gi" {% endif %} -{% if rabbitmq_cpu_limit is defined %} - cpu: "{{ rabbitmq_cpu_limit }}m" +{% if redis_cpu_limit is defined %} + cpu: "{{ redis_cpu_limit }}m" {% endif %} - name: {{ kubernetes_deployment_name }}-memcached image: "{{ kubernetes_memcached_image }}:{{ kubernetes_memcached_version }}" @@ -458,68 +289,6 @@ spec: - key: secret_key path: SECRET_KEY - - name: rabbitmq-config - configMap: - name: rabbitmq-config - items: - - key: rabbitmq.conf - path: rabbitmq.conf - - key: enabled_plugins - path: enabled_plugins - - key: rabbitmq_definitions.json - path: rabbitmq_definitions.json - - key: rabbitmq-env.conf - path: rabbitmq-env.conf - -{% if rabbitmq_use_ssl|default(False)|bool %} - - name: "{{ kubernetes_deployment_name }}-rabbitmq-certs-vol" - secret: - secretName: "{{ kubernetes_deployment_name }}-rabbitmq-certs" - items: - - key: rabbitmq_ssl_cert - path: 'server.crt' - - key: rabbitmq_ssl_key - path: 'server.key' - - key: rabbitmq_ssl_cacert - path: 'ca.crt' - - key: rabbitmq_ssl_combined - path: 'server-combined.pem' -{% endif %} - - name: rabbitmq-healthchecks - configMap: - name: {{ kubernetes_deployment_name }}-healthchecks - items: - - key: rabbit_health_node.py - path: rabbit_health_node.py - defaultMode: 0755 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ kubernetes_deployment_name }}-healthchecks - namespace: {{ kubernetes_namespace }} -data: - rabbit_health_node.py: | - #!/usr/bin/env python - try: - from http.client import HTTPConnection - except ImportError: - from httplib import HTTPConnection - import sys - import os - import base64 - authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD')) - conn=HTTPConnection('localhost:15672') - conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret}) - r1 = conn.getresponse() - if r1.status != 200: - sys.stderr.write('Received http error %i\\n' % (r1.status)) - sys.exit(1) - body = r1.read() - if body != '{"status":"ok"}': - sys.stderr.write('Received body: %s' % body) - sys.exit(2) - sys.exit(0) --- apiVersion: v1 kind: Service @@ -536,22 +305,7 @@ spec: targetPort: 8052 selector: name: {{ kubernetes_deployment_name }}-web-deploy ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ kubernetes_deployment_name }}-rmq-mgmt - namespace: {{ kubernetes_namespace }} - labels: - name: {{ kubernetes_deployment_name }}-rmq-mgmt -spec: - type: ClusterIP - ports: - - name: rmqmgmt - port: 15672 - targetPort: 15672 - selector: - name: {{ kubernetes_deployment_name }}-web-deploy + {% if kubernetes_context is defined %} --- apiVersion: extensions/v1beta1 diff --git a/installer/roles/kubernetes/templates/environment.sh.j2 b/installer/roles/kubernetes/templates/environment.sh.j2 index e10e7107b2..08c2608633 100644 --- a/installer/roles/kubernetes/templates/environment.sh.j2 +++ b/installer/roles/kubernetes/templates/environment.sh.j2 @@ -5,5 +5,3 @@ DATABASE_PORT={{ pg_port|default('5432') }} DATABASE_PASSWORD={{ pg_password | quote }} MEMCACHED_HOST={{ memcached_hostname|default('localhost') }} MEMCACHED_PORT={{ memcached_port|default('11211') }} -RABBITMQ_HOST={{ rabbitmq_hostname|default('localhost') }} -RABBITMQ_PORT={{ rabbitmq_port|default('5672') }} diff --git a/installer/roles/kubernetes/templates/management-pod.yml.j2 b/installer/roles/kubernetes/templates/management-pod.yml.j2 index 14d8b61cc5..0aaa2ab007 100644 --- a/installer/roles/kubernetes/templates/management-pod.yml.j2 +++ b/installer/roles/kubernetes/templates/management-pod.yml.j2 @@ -12,6 +12,7 @@ spec: containers: - name: ansible-tower-management image: "{{ kubernetes_task_image }}:{{ kubernetes_task_version }}" + imagePullPolicy: Always command: ["sleep", "infinity"] volumeMounts: - name: {{ kubernetes_deployment_name }}-application-config diff --git a/installer/roles/kubernetes/templates/secret.yml.j2 b/installer/roles/kubernetes/templates/secret.yml.j2 index 799f1adb57..989cb4485f 100644 --- a/installer/roles/kubernetes/templates/secret.yml.j2 +++ b/installer/roles/kubernetes/templates/secret.yml.j2 @@ -7,22 +7,5 @@ metadata: type: Opaque data: secret_key: "{{ secret_key | b64encode }}" - rabbitmq_password: "{{ rabbitmq_password | b64encode }}" - rabbitmq_erlang_cookie: "{{ rabbitmq_erlang_cookie | b64encode }}" credentials_py: "{{ lookup('template', 'credentials.py.j2') | b64encode }}" environment_sh: "{{ lookup('template', 'environment.sh.j2') | b64encode }}" - -{% if rabbitmq_use_ssl|default(False)|bool %} ---- -apiVersion: v1 -kind: Secret -metadata: - namespace: {{ kubernetes_namespace }} - name: "{{ kubernetes_deployment_name }}-rabbitmq-certs" -type: Opaque -data: - rabbitmq_ssl_cert: "{{ lookup('file', rmq_cert_tempdir.path + '/server.crt') | b64encode }}" - rabbitmq_ssl_key: "{{ lookup('file', rmq_cert_tempdir.path + '/server.key') | b64encode }}" - rabbitmq_ssl_cacert: "{{ lookup('file', rmq_cert_tempdir.path + '/ca.crt') | b64encode }}" - rabbitmq_ssl_combined: "{{ lookup('file', rmq_cert_tempdir.path + '/server-combined.pem') | b64encode }}" -{% endif %} From c06b6306abe295ae9e176e64c6ce35e1314438aa Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 14 Feb 2020 08:59:52 -0500 Subject: [PATCH 17/41] remove health info * Sending health about websockets over websockets is not a great idea. * I tried sending health data via prometheus and encountered problems that will need PR's to prometheus_client library to solve. Circle back to this later. --- awx/main/channels.py | 139 +----------------------------------------- awx/main/consumers.py | 23 ------- awx/main/routing.py | 1 - 3 files changed, 3 insertions(+), 160 deletions(-) diff --git a/awx/main/channels.py b/awx/main/channels.py index 5673166bb9..60bde215c2 100644 --- a/awx/main/channels.py +++ b/awx/main/channels.py @@ -18,14 +18,6 @@ from django.core.serializers.json import DjangoJSONEncoder logger = logging.getLogger('awx.main') -def dt_to_seconds(dt): - return int((dt - datetime.datetime(1970,1,1)).total_seconds()) - - -def now_seconds(): - return dt_to_seconds(datetime.datetime.now()) - - def wrap_broadcast_msg(group, message: str): # TODO: Maybe wrap as "group","message" so that we don't need to # encode/decode as json. @@ -51,64 +43,6 @@ def get_local_host(): return Instance.objects.me().hostname -# Second granularity; Per-minute -class FixedSlidingWindow(): - def __init__(self, start_time=None): - self.buckets = dict() - self.start_time = start_time or now_seconds() - - def cleanup(self, now_bucket=now_seconds()): - if self.start_time + 60 >= now_bucket: - self.start_time = now_bucket - 60 + 1 - - # Delete old entries - for k,v in self.buckets.items(): - if k < self.start_time: - del self.buckets[k] - - def record(self, ts=datetime.datetime.now()): - now_bucket = int((ts-datetime.datetime(1970,1,1)).total_seconds()) - - val = self.buckets.get(now_bucket, 0) - self.buckets[now_bucket] = val + 1 - - self.cleanup(now_bucket) - - def sum(self): - self.cleanup() - return sum(self.buckets.values()) or 0 - - -class Stats(): - def __init__(self, name): - self.name = name - self._messages_received_per_minute = FixedSlidingWindow() - self._messages_received = 0 - self._is_connected = False - self._connection_established_ts = None - - def record_message_received(self, ts=datetime.datetime.now()): - self._messages_received += 1 - self._messages_received_per_minute.record(ts) - - def get_messages_received_total(self): - return self._messages_received - - def get_messages_received_per_minute(self): - self._messages_received_per_minute.sum() - - def record_connection_established(self, ts=datetime.datetime.now()): - self._connection_established_ts = ts - - def record_connection_lost(self, ts=datetime.datetime.now()): - self._connection_established_ts = None - self._is_connected = False - - def get_connection_duration(self): - return (datetime.datetime.now() - self._connection_established_ts).total_seconds() - - - class WebsocketTask(): def __init__(self, name, @@ -119,7 +53,6 @@ class WebsocketTask(): verify_ssl: bool=settings.BROADCAST_WEBSOCKETS_VERIFY_CERT, endpoint: str='broadcast'): self.name = name - self.stats = Stats(name) self.event_loop = event_loop self.remote_host = remote_host self.remote_port = remote_port @@ -152,7 +85,6 @@ class WebsocketTask(): async with aiohttp.ClientSession(headers={'secret': secret_val}, timeout=timeout) as session: async with session.ws_connect(uri, ssl=self.verify_ssl) as websocket: - self.stats.record_connection_established() attempt = 0 await self.run_loop(websocket) except Exception as e: @@ -160,7 +92,6 @@ class WebsocketTask(): # Does aiohttp throws an exception if a disconnect happens? logger.warn("Websocket broadcast client exception {}".format(e)) finally: - self.stats.record_connection_lost() # Reconnect self.start(attempt=attempt+1) @@ -168,71 +99,9 @@ class WebsocketTask(): self.event_loop.create_task(self.connect(attempt=attempt)) -class HealthWebsocketTask(WebsocketTask): - def __init__(self, *args, **kwargs): - self.period = kwargs.pop('period', 10) - self.broadcast_stats = kwargs.pop('broadcast_stats', []) - - super().__init__(*args, endpoint='health', **kwargs) - - self.period_abs = None - # Ideally, we send a health beat at exactly the period. In reality - # there is always jitter due to OS needs, system load, etc. - # This variable tracks that offset. - self.last_period_offset = 0 - - async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse): - ''' - now = datetime.datetime.now() - if not self.period_abs: - self.period_abs = now - - sleep_time = self.period_abs + self.period - - - if now <= next_period: - logger.warn("Websocket broadcast missed sending health ping.") - else: - await asyncio.sleep(sleep_time) - - - sleep_time = datetime.datetime.now() - (self.last_period + datetime.timedelta(seconds=PERIOD)) - ''' - - # Start stats loop - self.event_loop.create_task(self.run_calc_stats_loop()) - - # Let this task loop be the send loop - await self.run_send_stats_loop(websocket) - - async def run_calc_stats_loop(self): - """ - Do any periodic calculations needed. i.e. sampling - """ - await asyncio.sleep(1) - - async def run_send_stats_loop(self, websocket: aiohttp.ClientWebSocketResponse): - while True: - msg = { - "sending_host": self.name, - "remote_hosts": [], - } - for s in self.broadcast_stats: - msg['remote_hosts'].append({ - 'name': s.name, - 'messages_received': s.get_messages_received_total(), - 'messages_received_per_minute': s.get_messages_received_per_minute(), - }) - - logger.debug(f"Sending health message {msg}") - await websocket.send_json(msg) - await asyncio.sleep(10) - - class BroadcastWebsocketTask(WebsocketTask): async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse): async for msg in websocket: - self.stats.record_message_received() if msg.type == aiohttp.WSMsgType.ERROR: break @@ -260,10 +129,8 @@ class RedisGroupBroadcastChannelLayer(RedisChannelLayer): loop = asyncio.get_event_loop() local_hostname = get_local_host() - broadcast_tasks = [BroadcastWebsocketTask(local_hostname, loop, h) for h in remote_hosts] - broadcast_stats = [t.stats for t in broadcast_tasks] - health_tasks = [HealthWebsocketTask(local_hostname, loop, h, broadcast_stats=broadcast_stats) for h in remote_hosts] + broadcast_tasks = [BroadcastWebsocketTask(name=local_hostname, + event_loop=loop, + remote_host=h) for h in remote_hosts] [t.start() for t in broadcast_tasks] - [t.start() for t in health_tasks] - diff --git a/awx/main/consumers.py b/awx/main/consumers.py index e88d9a603d..e11a20bc99 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -122,29 +122,6 @@ class BroadcastConsumer(AsyncJsonWebsocketConsumer): await self.send(event['text']) -class HealthConsumer(AsyncJsonWebsocketConsumer): - async def connect(self): - try: - WebsocketSecretAuthHelper.is_authorized(self.scope) - except Exception: - await self.close() - return - - # TODO: log ip of connected client - logger.info("Client connected to health endpoint") - await self.accept() - - async def disconnect(self, code): - # TODO: log ip of disconnected client - logger.info("Client disconnected from health endpoint") - - async def receive_json(self, content, **kwargs): - logger.debug(f"Got Health status {content}") - - async def internal_message(self, event): - logger.info("Got internal message from health endpoint .. can this happen?") - - class EventConsumer(AsyncJsonWebsocketConsumer): async def connect(self): user = self.scope['user'] diff --git a/awx/main/routing.py b/awx/main/routing.py index 38ba02a890..1efb6159d3 100644 --- a/awx/main/routing.py +++ b/awx/main/routing.py @@ -7,7 +7,6 @@ from . import consumers websocket_urlpatterns = [ url(r'websocket/$', consumers.EventConsumer), url(r'websocket/broadcast/$', consumers.BroadcastConsumer), - url(r'websocket/health/$', consumers.HealthConsumer), ] application = ProtocolTypeRouter({ From 03b73027e8781a62aec99f8c260b13f19e81c7da Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 14 Feb 2020 11:28:46 -0500 Subject: [PATCH 18/41] websockets aware of Instance changes * New tower nodes that are (de)registered in the Instance table are seen by the websocket layer and connected to or disconnected from by the websocket broadcast backplane using a polling mechanism. * This is especially useful for openshift and kubernetes. This will be useful for standalone Tower in the future when the restarting of Tower services is not required. --- awx/main/channels.py | 67 ++++++++++++++++++++++++++++++++-------- awx/settings/defaults.py | 3 ++ 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/awx/main/channels.py b/awx/main/channels.py index 60bde215c2..3d287d6622 100644 --- a/awx/main/channels.py +++ b/awx/main/channels.py @@ -75,8 +75,13 @@ class WebsocketTask(): if not self.channel_layer: self.channel_layer = get_channel_layer() - if attempt > 0: - await asyncio.sleep(5) + try: + if attempt > 0: + await asyncio.sleep(5) + except asyncio.CancelledError: + logger.warn(f"{self.name} connection to {self.remote_host} cancelled") + raise + uri = f"{self.protocol}://{self.remote_host}:{self.remote_port}/websocket/{self.endpoint}/" timeout = aiohttp.ClientTimeout(total=10) @@ -87,16 +92,23 @@ class WebsocketTask(): async with session.ws_connect(uri, ssl=self.verify_ssl) as websocket: attempt = 0 await self.run_loop(websocket) + except asyncio.CancelledError: + # TODO: Check if connected and disconnect + # Possibly use run_until_complete() if disconnect is async + logger.warn(f"{self.name} connection to {self.remote_host} cancelled") + raise except Exception as e: # Early on, this is our canary. I'm not sure what exceptions we can really encounter. # Does aiohttp throws an exception if a disconnect happens? logger.warn("Websocket broadcast client exception {}".format(e)) - finally: # Reconnect self.start(attempt=attempt+1) def start(self, attempt=0): - self.event_loop.create_task(self.connect(attempt=attempt)) + self.async_task = self.event_loop.create_task(self.connect(attempt=attempt)) + + def cancel(self): + self.async_task.cancel() class BroadcastWebsocketTask(WebsocketTask): @@ -121,16 +133,45 @@ class BroadcastWebsocketTask(WebsocketTask): await self.channel_layer.group_send(group, {"type": "internal.message", "text": message}) +class BroadcastWebsocketManager(object): + def __init__(self): + self.event_loop = asyncio.get_event_loop() + self.broadcast_tasks = dict() + + async def run_loop(self): + local_hostname = get_local_host() + + while True: + future_remote_hosts = get_broadcast_hosts() + current_remote_hosts = self.broadcast_tasks.keys() + deleted_remote_hosts = set(current_remote_hosts) - set(future_remote_hosts) + new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts) + + if deleted_remote_hosts: + logger.warn(f"{local_hostname} going to remove {deleted_remote_hosts} from the websocket broadcast list") + if new_remote_hosts: + logger.warn(f"{local_hostname} going to add {new_remote_hosts} to the websocket broadcast list") + + for h in deleted_remote_hosts: + self.broadcast_tasks[h].cancel() + del self.broadcast_tasks[h] + + for h in new_remote_hosts: + broadcast_task = BroadcastWebsocketTask(name=local_hostname, + event_loop=self.event_loop, + remote_host=h) + broadcast_task.start() + self.broadcast_tasks[h] = broadcast_task + + await asyncio.sleep(settings.BROADCAST_WEBSOCKETS_NEW_INSTANCE_POLL_RATE_SECONDS) + + def start(self): + self.async_task = self.event_loop.create_task(self.run_loop()) + + class RedisGroupBroadcastChannelLayer(RedisChannelLayer): def __init__(self, *args, **kwargs): super(RedisGroupBroadcastChannelLayer, self).__init__(*args, **kwargs) - remote_hosts = get_broadcast_hosts() - loop = asyncio.get_event_loop() - local_hostname = get_local_host() - - broadcast_tasks = [BroadcastWebsocketTask(name=local_hostname, - event_loop=loop, - remote_host=h) for h in remote_hosts] - - [t.start() for t in broadcast_tasks] + broadcast_websocket_mgr = BroadcastWebsocketManager() + broadcast_websocket_mgr.start() diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 798f8ba131..def4b9135e 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1250,3 +1250,6 @@ BROADCAST_WEBSOCKETS_VERIFY_CERT = False # Connect to other AWX nodes using http or https BROADCAST_WEBSOCKETS_PROTOCOL = 'https' + +# How often websocket process will look for changes in the Instance table +BROADCAST_WEBSOCKETS_NEW_INSTANCE_POLL_RATE_SECONDS = 10 From f5193e5ea5e906cf217bc9e33069702983673216 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 14 Feb 2020 14:36:19 -0500 Subject: [PATCH 19/41] resolve rebase errors --- awx/main/management/commands/run_dispatcher.py | 2 +- requirements/requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py index d12b23f275..c6d4bc71dc 100644 --- a/awx/main/management/commands/run_dispatcher.py +++ b/awx/main/management/commands/run_dispatcher.py @@ -11,7 +11,7 @@ from awx.main.utils.handlers import AWXProxyHandler from awx.main.dispatch import get_local_queuename, reaper from awx.main.dispatch.control import Control from awx.main.dispatch.pool import AutoscalePool -from awx.main.dispatch.worker import AWXConsumer, TaskWorker +from awx.main.dispatch.worker import AWXConsumerPG, TaskWorker from awx.main.dispatch import periodic logger = logging.getLogger('awx.main.dispatch') diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 8366788ebc..86218efd07 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -101,7 +101,6 @@ python-string-utils==0.6.0 # via openshift python3-openid==3.1.0 # via social-auth-core python3-saml==1.9.0 pytz==2019.3 # via django, irc, tempora, twilio -pyyaml==5.2 # via ansible-runner, djangorestframework-yaml, kubernetes pyyaml==5.3 # via ansible-runner, djangorestframework-yaml, kubernetes redis==3.3.11 requests-futures==1.0.0 From 3c5c9c6fde893327fb52614994b0d14d775b4ac1 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 14 Feb 2020 16:12:04 -0500 Subject: [PATCH 20/41] move broadcast websocket out into its own process --- Makefile | 33 ++----------------- awx/main/consumers.py | 6 ++-- awx/main/db/profiled_pg/base.py | 2 +- .../management/commands/run_wsbroadcast.py | 27 +++++++++++++++ awx/main/{channels.py => wsbroadcast.py} | 15 +++------ awx/settings/defaults.py | 2 +- awx/settings/local_settings.py.docker_compose | 2 +- docs/websockets.md | 6 ++-- installer/roles/image_build/files/settings.py | 5 ++- .../roles/image_build/files/supervisor.conf | 13 +++++++- .../kubernetes/templates/credentials.py.j2 | 4 +-- .../local_docker/templates/credentials.py.j2 | 2 +- tools/docker-compose/supervisor.conf | 16 +++++++++ 13 files changed, 76 insertions(+), 57 deletions(-) create mode 100644 awx/main/management/commands/run_wsbroadcast.py rename awx/main/{channels.py => wsbroadcast.py} (93%) diff --git a/Makefile b/Makefile index 16225f0454..dbf278f30d 100644 --- a/Makefile +++ b/Makefile @@ -265,28 +265,6 @@ migrate: dbchange: $(MANAGEMENT_COMMAND) makemigrations -server_noattach: - tmux new-session -d -s awx 'exec make uwsgi' - tmux rename-window 'AWX' - tmux select-window -t awx:0 - tmux split-window -v 'exec make dispatcher' - tmux new-window 'exec make daphne' - tmux select-window -t awx:1 - tmux rename-window 'WebSockets' - tmux split-window -h 'exec make runworker' - tmux split-window -v 'exec make nginx' - tmux new-window 'exec make receiver' - tmux select-window -t awx:2 - tmux rename-window 'Extra Services' - tmux select-window -t awx:0 - -server: server_noattach - tmux -2 attach-session -t awx - -# Use with iterm2's native tmux protocol support -servercc: server_noattach - tmux -2 -CC attach-session -t awx - supervisor: @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/awx/bin/activate; \ @@ -311,18 +289,11 @@ daphne: fi; \ daphne -b 127.0.0.1 -p 8051 awx.asgi:channel_layer -runworker: +wsbroadcast: @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/awx/bin/activate; \ fi; \ - $(PYTHON) manage.py runworker --only-channels websocket.* - -# Run the built-in development webserver (by default on http://localhost:8013). -runserver: - @if [ "$(VENV_BASE)" ]; then \ - . $(VENV_BASE)/awx/bin/activate; \ - fi; \ - $(PYTHON) manage.py runserver + $(PYTHON) manage.py run_wsbroadcast # Run to start the background task dispatcher for development. dispatcher: diff --git a/awx/main/consumers.py b/awx/main/consumers.py index e11a20bc99..68758210e8 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -21,7 +21,7 @@ from channels.db import database_sync_to_async from asgiref.sync import async_to_sync -from awx.main.channels import wrap_broadcast_msg +from awx.main.wsbroadcast import wrap_broadcast_msg logger = logging.getLogger('awx.main.consumers') @@ -106,11 +106,13 @@ class BroadcastConsumer(AsyncJsonWebsocketConsumer): try: WebsocketSecretAuthHelper.is_authorized(self.scope) except Exception: + # TODO: log ip of connected client + logger.warn("Broadcast client failed to authorize.") await self.close() return # TODO: log ip of connected client - logger.info("Client connected") + logger.info(f"Broadcast client connected.") await self.accept() await self.channel_layer.group_add(BROADCAST_GROUP, self.channel_name) diff --git a/awx/main/db/profiled_pg/base.py b/awx/main/db/profiled_pg/base.py index 820e867508..ab1f9a7c93 100644 --- a/awx/main/db/profiled_pg/base.py +++ b/awx/main/db/profiled_pg/base.py @@ -64,7 +64,7 @@ class RecordedQueryLog(object): if not os.path.isdir(self.dest): os.makedirs(self.dest) progname = ' '.join(sys.argv) - for match in ('uwsgi', 'dispatcher', 'callback_receiver', 'runworker'): + for match in ('uwsgi', 'dispatcher', 'callback_receiver', 'wsbroadcast'): if match in progname: progname = match break diff --git a/awx/main/management/commands/run_wsbroadcast.py b/awx/main/management/commands/run_wsbroadcast.py new file mode 100644 index 0000000000..63e8ed6b94 --- /dev/null +++ b/awx/main/management/commands/run_wsbroadcast.py @@ -0,0 +1,27 @@ +# Copyright (c) 2015 Ansible, Inc. +# All Rights Reserved. +import logging +import asyncio + +from django.core.management.base import BaseCommand + +from awx.main.wsbroadcast import BroadcastWebsocketManager + + +logger = logging.getLogger('awx.main.wsbroadcast') + + +class Command(BaseCommand): + help = 'Launch the websocket broadcaster' + + def handle(self, *arg, **options): + try: + broadcast_websocket_mgr = BroadcastWebsocketManager() + task = broadcast_websocket_mgr.start() + + loop = asyncio.get_event_loop() + loop.run_until_complete(task) + except KeyboardInterrupt: + logger.debug('Terminating Websocket Broadcaster') + if broadcast_websocket_mgr: + broadcast_websocket_mgr.stop() diff --git a/awx/main/channels.py b/awx/main/wsbroadcast.py similarity index 93% rename from awx/main/channels.py rename to awx/main/wsbroadcast.py index 3d287d6622..d5100793e4 100644 --- a/awx/main/channels.py +++ b/awx/main/wsbroadcast.py @@ -5,6 +5,7 @@ import logging import aiohttp import asyncio import datetime +import sys from channels_redis.core import RedisChannelLayer from channels.layers import get_channel_layer @@ -15,7 +16,7 @@ from django.apps import apps from django.core.serializers.json import DjangoJSONEncoder -logger = logging.getLogger('awx.main') +logger = logging.getLogger('awx.main.wsbroadcast') def wrap_broadcast_msg(group, message: str): @@ -100,7 +101,7 @@ class WebsocketTask(): except Exception as e: # Early on, this is our canary. I'm not sure what exceptions we can really encounter. # Does aiohttp throws an exception if a disconnect happens? - logger.warn("Websocket broadcast client exception {}".format(e)) + logger.warn(f"Websocket broadcast client exception {str(e)}") # Reconnect self.start(attempt=attempt+1) @@ -129,7 +130,6 @@ class BroadcastWebsocketTask(WebsocketTask): (group, message) = unwrap_broadcast_msg(payload) - logger.debug(f"{self.name} broadcasting message") await self.channel_layer.group_send(group, {"type": "internal.message", "text": message}) @@ -167,11 +167,4 @@ class BroadcastWebsocketManager(object): def start(self): self.async_task = self.event_loop.create_task(self.run_loop()) - - -class RedisGroupBroadcastChannelLayer(RedisChannelLayer): - def __init__(self, *args, **kwargs): - super(RedisGroupBroadcastChannelLayer, self).__init__(*args, **kwargs) - - broadcast_websocket_mgr = BroadcastWebsocketManager() - broadcast_websocket_mgr.start() + return self.async_task diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index def4b9135e..664259fb2d 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -959,7 +959,7 @@ ASGI_APPLICATION = "awx.main.routing.application" CHANNEL_LAYERS = { "default": { - "BACKEND": "awx.main.channels.RedisGroupBroadcastChannelLayer", + "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { "hosts": [("localhost", 6379)], }, diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index e2ae322e3d..9397d3c15e 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -58,7 +58,7 @@ redis_parts = parse.urlparse(BROKER_URL) CHANNEL_LAYERS = { "default": { - "BACKEND": "awx.main.channels.RedisGroupBroadcastChannelLayer", + "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { "hosts": [(redis_parts.hostname, redis_parts.port)] }, diff --git a/docs/websockets.md b/docs/websockets.md index 2095905028..a1f4403ee8 100644 --- a/docs/websockets.md +++ b/docs/websockets.md @@ -43,14 +43,14 @@ This section will specifically discuss deployment in the context of websockets a | `nginx` | listens on ports 80/443, handles HTTPS proxying, serves static assets, routes requests for `daphne` and `uwsgi` | | `uwsgi` | listens on port 8050, handles API requests | | `daphne` | listens on port 8051, handles websocket requests | -| `runworker` | no listening port, watches and processes the message queue | +| `wsbroadcast` | no listening port, forwards all group messages to all cluster nodes | | `supervisord` | (production-only) handles the process management of all the services except `nginx` | When a request comes in to `nginx` and has the `Upgrade` header and is for the path `/websocket`, then `nginx` knows that it should be routing that request to our `daphne` service. -`daphne` receives the request and generates channel and routing information for the request. The configured event handlers for `daphne` then unpack and parse the request message using the wire protocol mentioned above. This ensures that the connection has its context limited to only receive messages for events it is interested in. `daphne` uses internal events to trigger further behavior, which will generate messages and send them to the queue, which is then processed by the `runworker`. +`daphne` handles websocket connections proxied by nginx. -`runworker` processes the messages from the queue. This uses the contextual information of the message provided by the `daphne` server and our `asgi_amqp` implementation to broadcast messages out to each client. +`wsbroadcast` fully connects all cluster nodes to every other cluster nodes. Sends a copy of all group websocket messages to all other cluster nodes (i.e. job event type messages). ### Development - `nginx` listens on 8013/8043 instead of 80/443 diff --git a/installer/roles/image_build/files/settings.py b/installer/roles/image_build/files/settings.py index 6fe4306ec0..f33100fdef 100644 --- a/installer/roles/image_build/files/settings.py +++ b/installer/roles/image_build/files/settings.py @@ -87,11 +87,10 @@ if os.getenv("DATABASE_SSLMODE", False): BROKER_URL = 'redis://{}:{}'.format( os.getenv("REDIS_HOST", None), - os.getenv("REDIS_PORT", "6379"), + os.getenv("REDIS_PORT", "6379"),) CHANNEL_LAYERS = { - 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer', - 'ROUTING': 'awx.main.routing.channel_routing', + 'default': {'BACKEND': 'channels_redis.core.RedisChannelLayer', 'CONFIG': {'hosts': [(os.getenv("REDIS_HOST", None), int(os.getenv("REDIS_PORT", 6379)))]}} } diff --git a/installer/roles/image_build/files/supervisor.conf b/installer/roles/image_build/files/supervisor.conf index 1409b8b2c2..acc1af1d6b 100644 --- a/installer/roles/image_build/files/supervisor.conf +++ b/installer/roles/image_build/files/supervisor.conf @@ -35,8 +35,19 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 +[program:wsbroadcast] +command = awx-manage run_wsbroadcast +directory = /var/lib/awx +autostart = true +autorestart = true +stopwaitsecs = 5 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + [group:tower-processes] -programs=nginx,uwsgi,daphne +programs=nginx,uwsgi,daphne,wsbroadcast priority=5 # TODO: Exit Handler diff --git a/installer/roles/kubernetes/templates/credentials.py.j2 b/installer/roles/kubernetes/templates/credentials.py.j2 index fd68abc523..c53bacb2ed 100644 --- a/installer/roles/kubernetes/templates/credentials.py.j2 +++ b/installer/roles/kubernetes/templates/credentials.py.j2 @@ -18,6 +18,6 @@ BROKER_URL = 'redis://{}:{}/'.format( "{{ kubernetes_redis_port }}",) CHANNEL_LAYERS = { - 'default': {'BACKEND': 'awx.main.channels.RedisGroupBroadcastChannelLayer', + 'default': {'BACKEND': 'channels_redis.core.RedisChannelLayer', 'CONFIG': {'hosts': [("{{ kubernetes_redis_hostname }}", {{ kubernetes_redis_port|int }})]}} -} \ No newline at end of file +} diff --git a/installer/roles/local_docker/templates/credentials.py.j2 b/installer/roles/local_docker/templates/credentials.py.j2 index be71a5dc4a..8711785642 100644 --- a/installer/roles/local_docker/templates/credentials.py.j2 +++ b/installer/roles/local_docker/templates/credentials.py.j2 @@ -15,7 +15,7 @@ BROKER_URL = 'redis://{}:{}/'.format( "{{ redis_port }}",) CHANNEL_LAYERS = { - 'default': {'BACKEND': 'awx.main.channels.RedisGroupBroadcastChannelLayer', + 'default': {'BACKEND': 'channels_redis.core.RedisChannelLayer', 'CONFIG': {'hosts': [("{{ redis_hostname }}", {{ redis_port|int }})]}} } diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf index c0e649950d..8eee142b79 100644 --- a/tools/docker-compose/supervisor.conf +++ b/tools/docker-compose/supervisor.conf @@ -27,6 +27,18 @@ redirect_stderr=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 +[program:awx-wsbroadcast] +command = make wsbroadcast +autostart = true +autorestart = true +stopwaitsecs = 1 +stopsignal=KILL +stopasgroup=true +killasgroup=true +redirect_stderr=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 + [program:awx-uwsgi] command = make uwsgi autostart = true @@ -44,6 +56,10 @@ command = make daphne autostart = true autorestart = true redirect_stderr=true +stopwaitsecs = 1 +stopsignal=KILL +stopasgroup=true +killasgroup=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 From 14320bc8e6c6c97baf29ba6faffca1e5ed6f0e04 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Mon, 17 Feb 2020 09:04:16 -0500 Subject: [PATCH 21/41] handle websocket unsubscribe * Do not return from blocking unsubscribe until _after_ putting the gotten unsubscribe message on the queue so that it can be read by the thread of execution that was unblocked. --- awxkit/awxkit/ws.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/ws.py b/awxkit/awxkit/ws.py index 11668641bd..ea34410b9a 100644 --- a/awxkit/awxkit/ws.py +++ b/awxkit/awxkit/ws.py @@ -210,11 +210,13 @@ class WSClient(object): self._should_subscribe_to_pending_job['events'] == 'project_update_events'): self._update_subscription(message['unified_job_id']) + ret = self._recv_queue.put(message) + # unsubscribe acknowledgement if 'groups_current' in message: self._pending_unsubscribe.set() - return self._recv_queue.put(message) + return ret def _update_subscription(self, job_id): subscription = dict(jobs=self._should_subscribe_to_pending_job['jobs']) From 3b9e67ed1ba41f8d1ba3fe52d19eee65f7e1e944 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Mon, 17 Feb 2020 10:40:45 -0500 Subject: [PATCH 22/41] remove channel group model * Websocket user session <-> group subscription membership now resides in Redis rather than the database. --- awx/main/migrations/0110_delete_channelgroup.py | 16 ++++++++++++++++ awx/main/models/__init__.py | 1 - awx/main/models/channels.py | 6 ------ awx/settings/defaults.py | 5 ----- 4 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 awx/main/migrations/0110_delete_channelgroup.py delete mode 100644 awx/main/models/channels.py diff --git a/awx/main/migrations/0110_delete_channelgroup.py b/awx/main/migrations/0110_delete_channelgroup.py new file mode 100644 index 0000000000..7f571e587a --- /dev/null +++ b/awx/main/migrations/0110_delete_channelgroup.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.8 on 2020-02-17 14:50 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0109_v370_instance_ip_address'), + ] + + operations = [ + migrations.DeleteModel( + name='ChannelGroup', + ), + ] diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 2dbe959511..672d5e481a 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -58,7 +58,6 @@ from awx.main.models.workflow import ( # noqa WorkflowJob, WorkflowJobNode, WorkflowJobOptions, WorkflowJobTemplate, WorkflowJobTemplateNode, WorkflowApproval, WorkflowApprovalTemplate, ) -from awx.main.models.channels import ChannelGroup # noqa from awx.api.versioning import reverse from awx.main.models.oauth import ( # noqa OAuth2AccessToken, OAuth2Application diff --git a/awx/main/models/channels.py b/awx/main/models/channels.py deleted file mode 100644 index bd4f9514ba..0000000000 --- a/awx/main/models/channels.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.db import models - - -class ChannelGroup(models.Model): - group = models.CharField(max_length=200, unique=True) - channels = models.TextField() diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 664259fb2d..7d00314b18 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -451,11 +451,6 @@ CELERYBEAT_SCHEDULE = { # 'isolated_heartbeat': set up at the end of production.py and development.py } -ASGI_AMQP = { - 'INIT_FUNC': 'awx.prepare_env', - 'MODEL': 'awx.main.models.channels.ChannelGroup', -} - # Django Caching Configuration CACHES = { 'default': { From 0da94ada2b2be0b85b41b78041049729b4704b7e Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 21 Feb 2020 10:02:47 -0500 Subject: [PATCH 23/41] add missing service name to dev env * Dev env was bringing the wsbroadcast service up but not under the tower-process dependency. This is cleaner. --- tools/docker-compose/supervisor.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf index 8eee142b79..6831d80203 100644 --- a/tools/docker-compose/supervisor.conf +++ b/tools/docker-compose/supervisor.conf @@ -72,7 +72,7 @@ stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 [group:tower-processes] -programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx +programs=awx-dispatcher,awx-receiver,awx-uwsgi,awx-daphne,awx-nginx,awx-wsbroadcast priority=5 [unix_http_server] From b6b9802f9eebdf2c082ab9447a5e17815c77dcda Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 21 Feb 2020 10:05:31 -0500 Subject: [PATCH 24/41] increase per-channel capacity * 100 is the default capacity for a channel. If the client doesn't read the socket fast enough, websocket messages can and will be lost. This increases the default to 10,000 --- awx/settings/defaults.py | 1 + awx/settings/local_settings.py.docker_compose | 3 ++- installer/roles/image_build/files/settings.py | 5 ++++- installer/roles/kubernetes/templates/credentials.py.j2 | 5 ++++- installer/roles/local_docker/templates/credentials.py.j2 | 5 ++++- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 7d00314b18..f464369b78 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -957,6 +957,7 @@ CHANNEL_LAYERS = { "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { "hosts": [("localhost", 6379)], + "capacity": 10000, }, }, } diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index 9397d3c15e..a22bbda98f 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -60,7 +60,8 @@ CHANNEL_LAYERS = { "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { - "hosts": [(redis_parts.hostname, redis_parts.port)] + "hosts": [(redis_parts.hostname, redis_parts.port)], + "capacity": 10000, }, }, } diff --git a/installer/roles/image_build/files/settings.py b/installer/roles/image_build/files/settings.py index f33100fdef..a26d71c5c9 100644 --- a/installer/roles/image_build/files/settings.py +++ b/installer/roles/image_build/files/settings.py @@ -91,7 +91,10 @@ BROKER_URL = 'redis://{}:{}'.format( CHANNEL_LAYERS = { 'default': {'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': {'hosts': [(os.getenv("REDIS_HOST", None), int(os.getenv("REDIS_PORT", 6379)))]}} + 'CONFIG': { + 'hosts': [(os.getenv("REDIS_HOST", None), int(os.getenv("REDIS_PORT", 6379)))] + 'capacity': 10000, + }} } USE_X_FORWARDED_PORT = True diff --git a/installer/roles/kubernetes/templates/credentials.py.j2 b/installer/roles/kubernetes/templates/credentials.py.j2 index c53bacb2ed..e76d3ad74b 100644 --- a/installer/roles/kubernetes/templates/credentials.py.j2 +++ b/installer/roles/kubernetes/templates/credentials.py.j2 @@ -19,5 +19,8 @@ BROKER_URL = 'redis://{}:{}/'.format( CHANNEL_LAYERS = { 'default': {'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': {'hosts': [("{{ kubernetes_redis_hostname }}", {{ kubernetes_redis_port|int }})]}} + 'CONFIG': { + 'hosts': [("{{ kubernetes_redis_hostname }}", {{ kubernetes_redis_port|int }})], + 'capacity': 10000, + }} } diff --git a/installer/roles/local_docker/templates/credentials.py.j2 b/installer/roles/local_docker/templates/credentials.py.j2 index 8711785642..53f2df6dd1 100644 --- a/installer/roles/local_docker/templates/credentials.py.j2 +++ b/installer/roles/local_docker/templates/credentials.py.j2 @@ -16,7 +16,10 @@ BROKER_URL = 'redis://{}:{}/'.format( CHANNEL_LAYERS = { 'default': {'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': {'hosts': [("{{ redis_hostname }}", {{ redis_port|int }})]}} + 'CONFIG': { + 'hosts': [("{{ redis_hostname }}", {{ redis_port|int }})], + 'capacity': 10000, + }} } CACHES = { From d6594ab60240e058d630d1353bb0e46ab76d3c0c Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 21 Feb 2020 10:21:41 -0500 Subject: [PATCH 25/41] add broadcast websocket metrics * Gather brroadcast websocket metrics and push them into redis every configurable seconds. * Pop metrics from redis in web view layer to display via the api on demand --- awx/api/urls/urls.py | 6 +- awx/api/views/metrics.py | 16 ++ awx/api/views/root.py | 1 + awx/main/analytics/broadcast_websocket.py | 165 ++++++++++++++++++ awx/main/consumers.py | 23 +-- .../management/commands/run_wsbroadcast.py | 2 - awx/main/wsbroadcast.py | 40 +++-- awx/settings/defaults.py | 19 +- awx/settings/local_settings.py.docker_compose | 8 +- 9 files changed, 246 insertions(+), 34 deletions(-) create mode 100644 awx/main/analytics/broadcast_websocket.py diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index ab7d61fd23..4e5aa5b2f5 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -34,7 +34,10 @@ from awx.api.views import ( OAuth2ApplicationDetail, ) -from awx.api.views.metrics import MetricsView +from awx.api.views.metrics import ( + MetricsView, + BroadcastWebsocketMetricsView, +) from .organization import urls as organization_urls from .user import urls as user_urls @@ -93,6 +96,7 @@ v2_urls = [ url(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'), url(r'^', include(oauth2_urls)), url(r'^metrics/$', MetricsView.as_view(), name='metrics_view'), + url(r'^broadcast_websocket_metrics/$', BroadcastWebsocketMetricsView.as_view(), name='broadcast_websocket_metrics_view'), url(r'^ping/$', ApiV2PingView.as_view(), name='api_v2_ping_view'), url(r'^config/$', ApiV2ConfigView.as_view(), name='api_v2_config_view'), url(r'^config/subscriptions/$', ApiV2SubscriptionView.as_view(), name='api_v2_subscription_view'), diff --git a/awx/api/views/metrics.py b/awx/api/views/metrics.py index 092e36efde..64948e6bd1 100644 --- a/awx/api/views/metrics.py +++ b/awx/api/views/metrics.py @@ -15,6 +15,7 @@ from rest_framework.exceptions import PermissionDenied # AWX # from awx.main.analytics import collectors from awx.main.analytics.metrics import metrics +from awx.main.analytics.broadcast_websocket import BroadcastWebsocketStatsManager from awx.api import renderers from awx.api.generics import ( @@ -39,3 +40,18 @@ class MetricsView(APIView): if (request.user.is_superuser or request.user.is_system_auditor): return Response(metrics().decode('UTF-8')) raise PermissionDenied() + +class BroadcastWebsocketMetricsView(APIView): + name = _('Broadcast Websockets') + swagger_topic = 'Broadcast Websockets' + + renderer_classes = [renderers.PlainTextRenderer, + renderers.PrometheusJSONRenderer, + renderers.BrowsableAPIRenderer,] + + def get(self, request): + ''' Show Metrics Details ''' + if (request.user.is_superuser or request.user.is_system_auditor): + stats_str = BroadcastWebsocketStatsManager.get_stats_sync() or b'' + return Response(stats_str.decode('UTF-8')) + raise PermissionDenied() diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 4a15936e9b..3ac0759530 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -107,6 +107,7 @@ class ApiVersionRootView(APIView): data['applications'] = reverse('api:o_auth2_application_list', request=request) data['tokens'] = reverse('api:o_auth2_token_list', request=request) data['metrics'] = reverse('api:metrics_view', request=request) + data['broadcast_websocket_metrics'] = reverse('api:broadcast_websocket_metrics_view', request=request) data['inventory'] = reverse('api:inventory_list', request=request) data['inventory_scripts'] = reverse('api:inventory_script_list', request=request) data['inventory_sources'] = reverse('api:inventory_source_list', request=request) diff --git a/awx/main/analytics/broadcast_websocket.py b/awx/main/analytics/broadcast_websocket.py new file mode 100644 index 0000000000..08596eb5b6 --- /dev/null +++ b/awx/main/analytics/broadcast_websocket.py @@ -0,0 +1,165 @@ +import datetime +import os +import asyncio +import logging +import json +import aioredis +import redis + +from prometheus_client import ( + generate_latest, + Gauge, + Counter, + Enum, + Histogram, + Enum, + CollectorRegistry, + parser, +) + +from django.conf import settings + + +BROADCAST_WEBSOCKET_REDIS_KEY_NAME = 'broadcast_websocket_stats' + + +logger = logging.getLogger('awx.main.analytics.broadcast_websocket') + + +def dt_to_seconds(dt): + return int((dt - datetime.datetime(1970,1,1)).total_seconds()) + + +def now_seconds(): + return dt_to_seconds(datetime.datetime.now()) + + +# Second granularity; Per-minute +class FixedSlidingWindow(): + def __init__(self, start_time=None): + self.buckets = dict() + self.start_time = start_time or now_seconds() + + def cleanup(self, now_bucket=now_seconds()): + if self.start_time + 60 >= now_bucket: + self.start_time = now_bucket - 60 + 1 + + # Delete old entries + for k,v in self.buckets.items(): + if k < self.start_time: + del self.buckets[k] + + def record(self, ts=datetime.datetime.now()): + now_bucket = int((ts-datetime.datetime(1970,1,1)).total_seconds()) + + val = self.buckets.get(now_bucket, 0) + self.buckets[now_bucket] = val + 1 + + self.cleanup(now_bucket) + + def render(self): + self.cleanup() + return sum(self.buckets.values()) or 0 + + +class BroadcastWebsocketStatsManager(): + def __init__(self, event_loop, local_hostname): + self._local_hostname = local_hostname + + self._event_loop = event_loop + self._stats = dict() + self._redis_key = BROADCAST_WEBSOCKET_REDIS_KEY_NAME + + def new_remote_host_stats(self, remote_hostname): + self._stats[remote_hostname] = BroadcastWebsocketStats(self._local_hostname, + remote_hostname) + return self._stats[remote_hostname] + + def delete_remote_host_stats(self, remote_hostname): + del self._stats[remote_hostname] + + async def run_loop(self): + try: + redis_conn = await aioredis.create_redis_pool(settings.BROKER_URL) + while True: + stats_data_str = ''.join(stat.serialize() for stat in self._stats.values()) + await redis_conn.set(self._redis_key, stats_data_str) + + await asyncio.sleep(settings.BROADCAST_WEBSOCKET_STATS_POLL_RATE_SECONDS) + except Exception as e: + logger.warn(e) + await asyncio.sleep(settings.BROADCAST_WEBSOCKET_STATS_POLL_RATE_SECONDS) + self.start() + + def start(self): + self.async_task = self._event_loop.create_task(self.run_loop()) + return self.async_task + + @classmethod + def get_stats_sync(cls): + ''' + Stringified verion of all the stats + ''' + redis_conn = redis.Redis.from_url(settings.BROKER_URL) + return redis_conn.get(BROADCAST_WEBSOCKET_REDIS_KEY_NAME) + + +class BroadcastWebsocketStats(): + def __init__(self, local_hostname, remote_hostname): + self._local_hostname = local_hostname + self._remote_hostname = remote_hostname + self._registry = CollectorRegistry() + + # TODO: More robust replacement + self.name = self._local_hostname.replace('-', '_') + self.remote_name = self._remote_hostname.replace('-', '_') + + self._messages_received_total = Counter(f'awx_{self.remote_name}_messages_received_total', + 'Number of messages received, to be forwarded, by the broadcast websocket system', + registry=self._registry) + self._messages_received = Gauge(f'awx_{self.remote_name}_messages_received', + 'Number of messages received, to be forwarded, by the broadcast websocket system, for the duration of the current connection', + registry=self._registry) + self._connection = Enum(f'awx_{self.remote_name}_connection', + 'Websocket broadcast connection', + states=['disconnected', 'connected'], + registry=self._registry) + self._connection_start = Gauge(f'awx_{self.remote_name}_connection_start', + 'Time the connection was established', + registry=self._registry) + + self._messages_received_per_minute = Gauge(f'awx_{self.remote_name}_messages_received_per_minute', + 'Messages received per minute', + registry=self._registry) + self._internal_messages_received_per_minute = FixedSlidingWindow() + + def unregister(self): + self._registry.unregister(f'awx_{self.remote_name}_messages_received') + self._registry.unregister(f'awx_{self.remote_name}_connection') + + def record_message_received(self): + self._internal_messages_received_per_minute.record() + self._messages_received.inc() + self._messages_received_total.inc() + + def record_connection_established(self): + self._connection.state('connected') + self._connection_start.set_to_current_time() + self._messages_received.set(0) + + def record_connection_lost(self): + self._connection.state('disconnected') + + def get_connection_duration(self): + return (datetime.datetime.now() - self._connection_established_ts).total_seconds() + + def render(self): + msgs_per_min = self._internal_messages_received_per_minute.render() + self._messages_received_per_minute.set(msgs_per_min) + + def serialize(self): + self.render() + + data = {} + registry_data = generate_latest(self._registry).decode('UTF-8') + return registry_data diff --git a/awx/main/consumers.py b/awx/main/consumers.py index 68758210e8..07ee6e042c 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -21,12 +21,9 @@ from channels.db import database_sync_to_async from asgiref.sync import async_to_sync -from awx.main.wsbroadcast import wrap_broadcast_msg - logger = logging.getLogger('awx.main.consumers') XRF_KEY = '_auth_user_xrf' -BROADCAST_GROUP = 'broadcast-group_send' class WebsocketSecretAuthHelper: @@ -42,12 +39,12 @@ class WebsocketSecretAuthHelper: def construct_secret(cls): nonce_serialized = "{}".format(int((datetime.datetime.utcnow()-datetime.datetime.fromtimestamp(0)).total_seconds())) payload_dict = { - 'secret': settings.BROADCAST_WEBSOCKETS_SECRET, + 'secret': settings.BROADCAST_WEBSOCKET_SECRET, 'nonce': nonce_serialized } payload_serialized = json.dumps(payload_dict) - secret_serialized = hmac.new(force_bytes(settings.BROADCAST_WEBSOCKETS_SECRET), + secret_serialized = hmac.new(force_bytes(settings.BROADCAST_WEBSOCKET_SECRET), msg=force_bytes(payload_serialized), digestmod='sha256').hexdigest() @@ -68,14 +65,14 @@ class WebsocketSecretAuthHelper: try: payload_expected = { - 'secret': settings.BROADCAST_WEBSOCKETS_SECRET, + 'secret': settings.BROADCAST_WEBSOCKET_SECRET, 'nonce': nonce_parsed, } payload_serialized = json.dumps(payload_expected) except Exception: raise ValueError("Failed to create hash to compare to secret.") - secret_serialized = hmac.new(force_bytes(settings.BROADCAST_WEBSOCKETS_SECRET), + secret_serialized = hmac.new(force_bytes(settings.BROADCAST_WEBSOCKET_SECRET), msg=force_bytes(payload_serialized), digestmod='sha256').hexdigest() @@ -114,7 +111,7 @@ class BroadcastConsumer(AsyncJsonWebsocketConsumer): # TODO: log ip of connected client logger.info(f"Broadcast client connected.") await self.accept() - await self.channel_layer.group_add(BROADCAST_GROUP, self.channel_name) + await self.channel_layer.group_add(settings.BROADCAST_WEBSOCKET_GROUP_NAME, self.channel_name) async def disconnect(self, code): # TODO: log ip of disconnected client @@ -185,7 +182,7 @@ class EventConsumer(AsyncJsonWebsocketConsumer): continue new_groups.add(name) else: - if group_name == BROADCAST_GROUP: + if group_name == settings.BROADCAST_WEBSOCKET_GROUP_NAME: logger.warn("Non-priveleged client asked to join broadcast group!") return @@ -235,6 +232,8 @@ def _dump_payload(payload): async def emit_channel_notification_async(group, payload): + from awx.main.wsbroadcast import wrap_broadcast_msg # noqa + payload_dumped = _dump_payload(payload) if payload_dumped is None: return @@ -249,7 +248,7 @@ async def emit_channel_notification_async(group, payload): ) await channel_layer.group_send( - BROADCAST_GROUP, + settings.BROADCAST_WEBSOCKET_GROUP_NAME, { "type": "internal.message", "text": wrap_broadcast_msg(group, payload_dumped), @@ -258,6 +257,8 @@ async def emit_channel_notification_async(group, payload): def emit_channel_notification(group, payload): + from awx.main.wsbroadcast import wrap_broadcast_msg # noqa + payload_dumped = _dump_payload(payload) if payload_dumped is None: return @@ -273,7 +274,7 @@ def emit_channel_notification(group, payload): )) run_sync(channel_layer.group_send( - BROADCAST_GROUP, + settings.BROADCAST_WEBSOCKET_GROUP_NAME, { "type": "internal.message", "text": wrap_broadcast_msg(group, payload_dumped), diff --git a/awx/main/management/commands/run_wsbroadcast.py b/awx/main/management/commands/run_wsbroadcast.py index 63e8ed6b94..cb684a3577 100644 --- a/awx/main/management/commands/run_wsbroadcast.py +++ b/awx/main/management/commands/run_wsbroadcast.py @@ -23,5 +23,3 @@ class Command(BaseCommand): loop.run_until_complete(task) except KeyboardInterrupt: logger.debug('Terminating Websocket Broadcaster') - if broadcast_websocket_mgr: - broadcast_websocket_mgr.stop() diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index d5100793e4..7aef604f52 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -15,6 +15,11 @@ from django.conf import settings from django.apps import apps from django.core.serializers.json import DjangoJSONEncoder +from awx.main.analytics.broadcast_websocket import ( + BroadcastWebsocketStats, + BroadcastWebsocketStatsManager, +) + logger = logging.getLogger('awx.main.wsbroadcast') @@ -48,13 +53,15 @@ class WebsocketTask(): def __init__(self, name, event_loop, + stats: BroadcastWebsocketStats, remote_host: str, - remote_port: int=settings.BROADCAST_WEBSOCKETS_PORT, - protocol: str=settings.BROADCAST_WEBSOCKETS_PROTOCOL, - verify_ssl: bool=settings.BROADCAST_WEBSOCKETS_VERIFY_CERT, + remote_port: int=settings.BROADCAST_WEBSOCKET_PORT, + protocol: str=settings.BROADCAST_WEBSOCKET_PROTOCOL, + verify_ssl: bool=settings.BROADCAST_WEBSOCKET_VERIFY_CERT, endpoint: str='broadcast'): self.name = name self.event_loop = event_loop + self.stats = stats self.remote_host = remote_host self.remote_port = remote_port self.endpoint = endpoint @@ -78,7 +85,7 @@ class WebsocketTask(): try: if attempt > 0: - await asyncio.sleep(5) + await asyncio.sleep(settings.BROADCAST_WEBSOCKET_RECONNECT_RETRY_RATE_SECONDS) except asyncio.CancelledError: logger.warn(f"{self.name} connection to {self.remote_host} cancelled") raise @@ -91,17 +98,20 @@ class WebsocketTask(): async with aiohttp.ClientSession(headers={'secret': secret_val}, timeout=timeout) as session: async with session.ws_connect(uri, ssl=self.verify_ssl) as websocket: + self.stats.record_connection_established() attempt = 0 await self.run_loop(websocket) except asyncio.CancelledError: # TODO: Check if connected and disconnect # Possibly use run_until_complete() if disconnect is async logger.warn(f"{self.name} connection to {self.remote_host} cancelled") + self.stats.record_connection_lost() raise except Exception as e: # Early on, this is our canary. I'm not sure what exceptions we can really encounter. # Does aiohttp throws an exception if a disconnect happens? logger.warn(f"Websocket broadcast client exception {str(e)}") + self.stats.record_connection_lost() # Reconnect self.start(attempt=attempt+1) @@ -115,6 +125,7 @@ class WebsocketTask(): class BroadcastWebsocketTask(WebsocketTask): async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse): async for msg in websocket: + self.stats.record_message_received() if msg.type == aiohttp.WSMsgType.ERROR: break @@ -137,9 +148,11 @@ class BroadcastWebsocketManager(object): def __init__(self): self.event_loop = asyncio.get_event_loop() self.broadcast_tasks = dict() + # parallel dict to broadcast_tasks that tracks stats + self.local_hostname = get_local_host() + self.stats_mgr = BroadcastWebsocketStatsManager(self.event_loop, self.local_hostname) - async def run_loop(self): - local_hostname = get_local_host() + async def run_per_host_websocket(self): while True: future_remote_hosts = get_broadcast_hosts() @@ -148,23 +161,28 @@ class BroadcastWebsocketManager(object): new_remote_hosts = set(future_remote_hosts) - set(current_remote_hosts) if deleted_remote_hosts: - logger.warn(f"{local_hostname} going to remove {deleted_remote_hosts} from the websocket broadcast list") + logger.warn(f"{self.local_hostname} going to remove {deleted_remote_hosts} from the websocket broadcast list") if new_remote_hosts: - logger.warn(f"{local_hostname} going to add {new_remote_hosts} to the websocket broadcast list") + logger.warn(f"{self.local_hostname} going to add {new_remote_hosts} to the websocket broadcast list") for h in deleted_remote_hosts: self.broadcast_tasks[h].cancel() del self.broadcast_tasks[h] + self.stats_mgr.delete_remote_host_stats(h) for h in new_remote_hosts: - broadcast_task = BroadcastWebsocketTask(name=local_hostname, + stats = self.stats_mgr.new_remote_host_stats(h) + broadcast_task = BroadcastWebsocketTask(name=self.local_hostname, event_loop=self.event_loop, + stats=stats, remote_host=h) broadcast_task.start() self.broadcast_tasks[h] = broadcast_task - await asyncio.sleep(settings.BROADCAST_WEBSOCKETS_NEW_INSTANCE_POLL_RATE_SECONDS) + await asyncio.sleep(settings.BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS) def start(self): - self.async_task = self.event_loop.create_task(self.run_loop()) + self.stats_mgr.start() + + self.async_task = self.event_loop.create_task(self.run_per_host_websocket()) return self.async_task diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index f464369b78..47d6a3522f 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1235,17 +1235,26 @@ MIDDLEWARE = [ # Secret header value to exchange for websockets responsible for distributing websocket messages. # This needs to be kept secret and randomly generated -BROADCAST_WEBSOCKETS_SECRET = '' +BROADCAST_WEBSOCKET_SECRET = '' # Port for broadcast websockets to connect to # Note: that the clients will follow redirect responses -BROADCAST_WEBSOCKETS_PORT = 443 +BROADCAST_WEBSOCKET_PORT = 443 # Whether or not broadcast websockets should check nginx certs when interconnecting -BROADCAST_WEBSOCKETS_VERIFY_CERT = False +BROADCAST_WEBSOCKET_VERIFY_CERT = False # Connect to other AWX nodes using http or https -BROADCAST_WEBSOCKETS_PROTOCOL = 'https' +BROADCAST_WEBSOCKET_PROTOCOL = 'https' + +# All websockets that connect to the broadcast websocket endpoint will be put into this group +BROADCAST_WEBSOCKET_GROUP_NAME = 'broadcast-group_send' + +# Time wait before retrying connecting to a websocket broadcast tower node +BROADCAST_WEBSOCKET_RECONNECT_RETRY_RATE_SECONDS = 5 # How often websocket process will look for changes in the Instance table -BROADCAST_WEBSOCKETS_NEW_INSTANCE_POLL_RATE_SECONDS = 10 +BROADCAST_WEBSOCKET_NEW_INSTANCE_POLL_RATE_SECONDS = 10 + +# How often websocket process will generate stats +BROADCAST_WEBSOCKET_STATS_POLL_RATE_SECONDS = 5 diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index a22bbda98f..8d785e62ef 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -244,7 +244,7 @@ TEST_OPENSTACK_PROJECT = '' TEST_AZURE_USERNAME = '' TEST_AZURE_KEY_DATA = '' -BROADCAST_WEBSOCKETS_SECRET = '🤖starscream🤖' -BROADCAST_WEBSOCKETS_PORT = 8013 -BROADCAST_WEBSOCKETS_VERIFY_CERT = False -BROADCAST_WEBSOCKETS_PROTOCOL = 'http' +BROADCAST_WEBSOCKET_SECRET = '🤖starscream🤖' +BROADCAST_WEBSOCKET_PORT = 8013 +BROADCAST_WEBSOCKET_VERIFY_CERT = False +BROADCAST_WEBSOCKET_PROTOCOL = 'http' From 8350bb3371b9eb62a49e5370feb138f1603c9653 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 21 Feb 2020 13:51:47 -0500 Subject: [PATCH 26/41] robust broadcast websocket error hanndling --- awx/main/wsbroadcast.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index 7aef604f52..f7171a8627 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -2,11 +2,13 @@ import os import json import logging -import aiohttp import asyncio import datetime import sys +import aiohttp +from aiohttp import client_exceptions + from channels_redis.core import RedisChannelLayer from channels.layers import get_channel_layer @@ -107,12 +109,18 @@ class WebsocketTask(): logger.warn(f"{self.name} connection to {self.remote_host} cancelled") self.stats.record_connection_lost() raise + except client_exceptions.ClientConnectorError as e: + logger.warn(f"Failed to connect to {self.remote_host}: '{e}'. Reconnecting ...") + self.stats.record_connection_lost() + self.start(attempt=attempt+1) + except asyncio.TimeoutError: + logger.warn(f"Timeout while trying to connect to {self.remote_host}. Reconnecting ...") + self.stats.record_connection_lost() + self.start(attempt=attempt+1) except Exception as e: # Early on, this is our canary. I'm not sure what exceptions we can really encounter. - # Does aiohttp throws an exception if a disconnect happens? - logger.warn(f"Websocket broadcast client exception {str(e)}") + logger.warn(f"Websocket broadcast client exception {type(e)} {e}") self.stats.record_connection_lost() - # Reconnect self.start(attempt=attempt+1) def start(self, attempt=0): From e25bd931a111ea85826d000709735fd0c7a66853 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 21 Feb 2020 15:02:53 -0500 Subject: [PATCH 27/41] change dispatcher test to make required queue * No fallback-default queue anymore. Queue must be explicitly provided. --- awx/main/tests/functional/test_dispatch.py | 24 +++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/awx/main/tests/functional/test_dispatch.py b/awx/main/tests/functional/test_dispatch.py index 97f1ccf1d6..c13a031af3 100644 --- a/awx/main/tests/functional/test_dispatch.py +++ b/awx/main/tests/functional/test_dispatch.py @@ -15,6 +15,15 @@ from awx.main.dispatch.publish import task from awx.main.dispatch.worker import BaseWorker, TaskWorker +''' +Prevent logger. calls from triggering database operations +''' +@pytest.fixture(autouse=True) +def _disable_database_settings(mocker): + m = mocker.patch('awx.conf.settings.SettingsWrapper.all_supported_settings', new_callable=mock.PropertyMock) + m.return_value = [] + + def restricted(a, b): raise AssertionError("This code should not run because it isn't decorated with @task") @@ -324,22 +333,23 @@ class TestTaskPublisher: assert Adder().run(2, 2) == 4 def test_function_apply_async(self): - message, queue = add.apply_async([2, 2]) + message, queue = add.apply_async([2, 2], queue='foobar') assert message['args'] == [2, 2] assert message['kwargs'] == {} assert message['task'] == 'awx.main.tests.functional.test_dispatch.add' - assert queue == 'awx_private_queue' + assert queue == 'foobar' def test_method_apply_async(self): - message, queue = Adder.apply_async([2, 2]) + message, queue = Adder.apply_async([2, 2], queue='foobar') assert message['args'] == [2, 2] assert message['kwargs'] == {} assert message['task'] == 'awx.main.tests.functional.test_dispatch.Adder' - assert queue == 'awx_private_queue' + assert queue == 'foobar' - def test_apply_with_queue(self): - message, queue = add.apply_async([2, 2], queue='abc123') - assert queue == 'abc123' + def test_apply_async_queue_required(self): + with pytest.raises(ValueError) as e: + message, queue = add.apply_async([2, 2]) + assert "awx.main.tests.functional.test_dispatch.add: Queue value required and may not me None" == e.value.args[0] def test_queue_defined_in_task_decorator(self): message, queue = multiply.apply_async([2, 2]) From 093d204d198e18af28db89a841bd60237f801cfe Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 21 Feb 2020 15:36:27 -0500 Subject: [PATCH 28/41] fix flake8 --- awx/api/views/metrics.py | 1 + awx/main/analytics/broadcast_websocket.py | 18 ++++++----------- awx/main/consumers.py | 18 +++-------------- awx/main/dispatch/__init__.py | 6 +++--- awx/main/dispatch/control.py | 1 - awx/main/dispatch/publish.py | 4 +--- awx/main/dispatch/worker/base.py | 1 - .../management/commands/run_dispatcher.py | 2 +- awx/main/queue.py | 1 - awx/main/routing.py | 1 - awx/main/wsbroadcast.py | 20 +++++++------------ awxkit/awxkit/ws.py | 3 --- 12 files changed, 22 insertions(+), 54 deletions(-) diff --git a/awx/api/views/metrics.py b/awx/api/views/metrics.py index 64948e6bd1..cc6a3e3994 100644 --- a/awx/api/views/metrics.py +++ b/awx/api/views/metrics.py @@ -41,6 +41,7 @@ class MetricsView(APIView): return Response(metrics().decode('UTF-8')) raise PermissionDenied() + class BroadcastWebsocketMetricsView(APIView): name = _('Broadcast Websockets') swagger_topic = 'Broadcast Websockets' diff --git a/awx/main/analytics/broadcast_websocket.py b/awx/main/analytics/broadcast_websocket.py index 08596eb5b6..5485e439c7 100644 --- a/awx/main/analytics/broadcast_websocket.py +++ b/awx/main/analytics/broadcast_websocket.py @@ -1,8 +1,6 @@ import datetime -import os import asyncio import logging -import json import aioredis import redis @@ -11,10 +9,7 @@ from prometheus_client import ( Gauge, Counter, Enum, - Histogram, - Enum, CollectorRegistry, - parser, ) from django.conf import settings @@ -50,7 +45,7 @@ class FixedSlidingWindow(): del self.buckets[k] def record(self, ts=datetime.datetime.now()): - now_bucket = int((ts-datetime.datetime(1970,1,1)).total_seconds()) + now_bucket = int((ts - datetime.datetime(1970,1,1)).total_seconds()) val = self.buckets.get(now_bucket, 0) self.buckets[now_bucket] = val + 1 @@ -118,15 +113,15 @@ class BroadcastWebsocketStats(): 'Number of messages received, to be forwarded, by the broadcast websocket system', registry=self._registry) self._messages_received = Gauge(f'awx_{self.remote_name}_messages_received', - 'Number of messages received, to be forwarded, by the broadcast websocket system, for the duration of the current connection', + 'Number forwarded messages received by the broadcast websocket system, for the duration of the current connection', registry=self._registry) self._connection = Enum(f'awx_{self.remote_name}_connection', - 'Websocket broadcast connection', - states=['disconnected', 'connected'], - registry=self._registry) + 'Websocket broadcast connection', + states=['disconnected', 'connected'], + registry=self._registry) self._connection_start = Gauge(f'awx_{self.remote_name}_connection_start', 'Time the connection was established', - registry=self._registry) + registry=self._registry) self._messages_received_per_minute = Gauge(f'awx_{self.remote_name}_messages_received_per_minute', 'Messages received per minute', @@ -160,6 +155,5 @@ class BroadcastWebsocketStats(): def serialize(self): self.render() - data = {} registry_data = generate_latest(self._registry).decode('UTF-8') return registry_data diff --git a/awx/main/consumers.py b/awx/main/consumers.py index 07ee6e042c..a14322cb1f 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -1,15 +1,9 @@ - -import os import json import logging -import codecs import datetime import hmac import asyncio -from django.utils.encoding import force_bytes -from django.utils.encoding import smart_str -from django.http.cookie import parse_cookie from django.core.serializers.json import DjangoJSONEncoder from django.conf import settings from django.utils.encoding import force_bytes @@ -19,8 +13,6 @@ from channels.generic.websocket import AsyncJsonWebsocketConsumer from channels.layers import get_channel_layer from channels.db import database_sync_to_async -from asgiref.sync import async_to_sync - logger = logging.getLogger('awx.main.consumers') XRF_KEY = '_auth_user_xrf' @@ -37,7 +29,7 @@ class WebsocketSecretAuthHelper: @classmethod def construct_secret(cls): - nonce_serialized = "{}".format(int((datetime.datetime.utcnow()-datetime.datetime.fromtimestamp(0)).total_seconds())) + nonce_serialized = "{}".format(int((datetime.datetime.utcnow() - datetime.datetime.fromtimestamp(0)).total_seconds())) payload_dict = { 'secret': settings.BROADCAST_WEBSOCKET_SECRET, 'nonce': nonce_serialized @@ -53,8 +45,6 @@ class WebsocketSecretAuthHelper: @classmethod def verify_secret(cls, s, nonce_tolerance=300): - hex_decoder = codecs.getdecoder("hex_codec") - try: (prefix, payload) = s.split(' ') if prefix != 'HMAC-SHA256': @@ -82,7 +72,7 @@ class WebsocketSecretAuthHelper: # Avoid timing attack and check the nonce after all the heavy lifting now = datetime.datetime.utcnow() nonce_parsed = datetime.datetime.fromtimestamp(int(nonce_parsed)) - if (now-nonce_parsed).total_seconds() > nonce_tolerance: + if (now - nonce_parsed).total_seconds() > nonce_tolerance: raise ValueError("Potential replay attack or machine(s) time out of sync.") return True @@ -160,9 +150,7 @@ class EventConsumer(AsyncJsonWebsocketConsumer): XRF_KEY not in self.scope["session"] or xrftoken != self.scope["session"][XRF_KEY] ): - logger.error( - "access denied to channel, XRF mismatch for {}".format(user.username) - ) + logger.error(f"access denied to channel, XRF mismatch for {user.username}") await self.send_json({"error": "access denied to channel"}) return diff --git a/awx/main/dispatch/__init__.py b/awx/main/dispatch/__init__.py index d97919dada..841f9344ae 100644 --- a/awx/main/dispatch/__init__.py +++ b/awx/main/dispatch/__init__.py @@ -73,9 +73,9 @@ class PubSub(object): def pg_bus_conn(): conf = settings.DATABASES['default'] conn = psycopg2.connect(dbname=conf['NAME'], - host=conf['HOST'], - user=conf['USER'], - password=conf['PASSWORD']) + host=conf['HOST'], + user=conf['USER'], + password=conf['PASSWORD']) # Django connection.cursor().connection doesn't have autocommit=True on conn.set_session(autocommit=True) pubsub = PubSub(conn) diff --git a/awx/main/dispatch/control.py b/awx/main/dispatch/control.py index 684cdae806..4565df17f5 100644 --- a/awx/main/dispatch/control.py +++ b/awx/main/dispatch/control.py @@ -1,5 +1,4 @@ import logging -import socket import string import random import json diff --git a/awx/main/dispatch/publish.py b/awx/main/dispatch/publish.py index 02fca647f6..020e7407cd 100644 --- a/awx/main/dispatch/publish.py +++ b/awx/main/dispatch/publish.py @@ -2,13 +2,11 @@ import inspect import logging import sys import json -import re from uuid import uuid4 from django.conf import settings -from django.db import connection -from . import pg_bus_conn, get_local_queuename +from . import pg_bus_conn logger = logging.getLogger('awx.main.dispatch') diff --git a/awx/main/dispatch/worker/base.py b/awx/main/dispatch/worker/base.py index 674892a7b5..ef6270ff90 100644 --- a/awx/main/dispatch/worker/base.py +++ b/awx/main/dispatch/worker/base.py @@ -7,7 +7,6 @@ import signal import sys import redis import json -import re import psycopg2 from uuid import UUID from queue import Empty as QueueEmpty diff --git a/awx/main/management/commands/run_dispatcher.py b/awx/main/management/commands/run_dispatcher.py index c6d4bc71dc..5f7db4f106 100644 --- a/awx/main/management/commands/run_dispatcher.py +++ b/awx/main/management/commands/run_dispatcher.py @@ -5,7 +5,7 @@ import logging from django.conf import settings from django.core.cache import cache as django_cache from django.core.management.base import BaseCommand -from django.db import connection as django_connection, connections +from django.db import connection as django_connection from awx.main.utils.handlers import AWXProxyHandler from awx.main.dispatch import get_local_queuename, reaper diff --git a/awx/main/queue.py b/awx/main/queue.py index 38bea6fc2c..762879fd2c 100644 --- a/awx/main/queue.py +++ b/awx/main/queue.py @@ -4,7 +4,6 @@ # Python import json import logging -import os import redis # Django diff --git a/awx/main/routing.py b/awx/main/routing.py index 1efb6159d3..090634cbb8 100644 --- a/awx/main/routing.py +++ b/awx/main/routing.py @@ -1,4 +1,3 @@ -from django.urls import re_path from django.conf.urls import url from channels.auth import AuthMiddlewareStack from channels.routing import ProtocolTypeRouter, URLRouter diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index f7171a8627..fefcc99682 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -1,18 +1,12 @@ - -import os import json import logging import asyncio -import datetime -import sys import aiohttp from aiohttp import client_exceptions -from channels_redis.core import RedisChannelLayer from channels.layers import get_channel_layer -from django.utils.encoding import force_bytes from django.conf import settings from django.apps import apps from django.core.serializers.json import DjangoJSONEncoder @@ -57,10 +51,10 @@ class WebsocketTask(): event_loop, stats: BroadcastWebsocketStats, remote_host: str, - remote_port: int=settings.BROADCAST_WEBSOCKET_PORT, - protocol: str=settings.BROADCAST_WEBSOCKET_PROTOCOL, - verify_ssl: bool=settings.BROADCAST_WEBSOCKET_VERIFY_CERT, - endpoint: str='broadcast'): + remote_port: int = settings.BROADCAST_WEBSOCKET_PORT, + protocol: str = settings.BROADCAST_WEBSOCKET_PROTOCOL, + verify_ssl: bool = settings.BROADCAST_WEBSOCKET_VERIFY_CERT, + endpoint: str = 'broadcast'): self.name = name self.event_loop = event_loop self.stats = stats @@ -112,16 +106,16 @@ class WebsocketTask(): except client_exceptions.ClientConnectorError as e: logger.warn(f"Failed to connect to {self.remote_host}: '{e}'. Reconnecting ...") self.stats.record_connection_lost() - self.start(attempt=attempt+1) + self.start(attempt=attempt + 1) except asyncio.TimeoutError: logger.warn(f"Timeout while trying to connect to {self.remote_host}. Reconnecting ...") self.stats.record_connection_lost() - self.start(attempt=attempt+1) + self.start(attempt=attempt + 1) except Exception as e: # Early on, this is our canary. I'm not sure what exceptions we can really encounter. logger.warn(f"Websocket broadcast client exception {type(e)} {e}") self.stats.record_connection_lost() - self.start(attempt=attempt+1) + self.start(attempt=attempt + 1) def start(self, attempt=0): self.async_task = self.event_loop.create_task(self.connect(attempt=attempt)) diff --git a/awxkit/awxkit/ws.py b/awxkit/awxkit/ws.py index ea34410b9a..486fd5dc4a 100644 --- a/awxkit/awxkit/ws.py +++ b/awxkit/awxkit/ws.py @@ -1,10 +1,8 @@ -import time import threading import logging import atexit import json import ssl -from datetime import datetime from six.moves.queue import Queue, Empty from six.moves.urllib.parse import urlparse @@ -187,7 +185,6 @@ class WSClient(object): self._send(json.dumps(payload)) def unsubscribe(self, wait=True, timeout=10): - time_start = datetime.now() if wait: # Other unnsubscribe events could have caused the edge to trigger. # This way the _next_ event will trigger our waiting. From 9e5fe7f5c68f0a7f375fb8c5ea683d402198a85f Mon Sep 17 00:00:00 2001 From: chris meyers Date: Sun, 23 Feb 2020 11:13:19 -0500 Subject: [PATCH 29/41] translate Instance hostname to safe analytics name * More robust translation of Instance hostname to analytics safe name by replacing all non-alpha numeric characters with _ --- awx/main/analytics/broadcast_websocket.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/awx/main/analytics/broadcast_websocket.py b/awx/main/analytics/broadcast_websocket.py index 5485e439c7..2b087b3dff 100644 --- a/awx/main/analytics/broadcast_websocket.py +++ b/awx/main/analytics/broadcast_websocket.py @@ -3,6 +3,7 @@ import asyncio import logging import aioredis import redis +import re from prometheus_client import ( generate_latest, @@ -106,8 +107,8 @@ class BroadcastWebsocketStats(): self._registry = CollectorRegistry() # TODO: More robust replacement - self.name = self._local_hostname.replace('-', '_') - self.remote_name = self._remote_hostname.replace('-', '_') + self.name = self.safe_name(self._local_hostname) + self.remote_name = self.safe_name(self._remote_hostname) self._messages_received_total = Counter(f'awx_{self.remote_name}_messages_received_total', 'Number of messages received, to be forwarded, by the broadcast websocket system', @@ -128,6 +129,10 @@ class BroadcastWebsocketStats(): registry=self._registry) self._internal_messages_received_per_minute = FixedSlidingWindow() + def safe_name(self, s): + # Replace all non alpha-numeric characters with _ + return re.sub('[^0-9a-zA-Z]+', '_', s) + def unregister(self): self._registry.unregister(f'awx_{self.remote_name}_messages_received') self._registry.unregister(f'awx_{self.remote_name}_connection') From 2b59af3808d8b110c2c430349aad9fe825bb3904 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Wed, 26 Feb 2020 11:50:22 -0500 Subject: [PATCH 30/41] safely operate in async or sync context --- awx/main/consumers.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/awx/main/consumers.py b/awx/main/consumers.py index a14322cb1f..00ffcadbd8 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -202,13 +202,9 @@ class EventConsumer(AsyncJsonWebsocketConsumer): def run_sync(func): - event_loop = None - try: - event_loop = asyncio.get_event_loop() - except RuntimeError: - event_loop = asyncio.new_event_loop() - asyncio.set_event_loop(event_loop) - return event_loop.run_until_complete(func) + event_loop = asyncio.new_event_loop() + event_loop.run_until_complete(func) + event_loop.close() def _dump_payload(payload): From 3f5e2a3cd3e80416d90b9133c6b27e1d1fec7109 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Thu, 27 Feb 2020 15:17:01 -0500 Subject: [PATCH 31/41] try to make openshift build happy --- requirements/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 86218efd07..c470456a5f 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -110,6 +110,7 @@ rsa==4.0 # via google-auth ruamel.yaml.clib==0.2.0 # via ruamel.yaml ruamel.yaml==0.16.5 # via openshift schedule==0.6.0 +service-identity==18.1.0 # via twisted six==1.13.0 # via ansible-runner, asgi-amqp, asgiref, autobahn, automat, cryptography, django-extensions, google-auth, isodate, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.text, jsonschema, kubernetes, openshift, pygerduty, pyhamcrest, pyrad, pyrsistent, python-dateutil, python-memcached, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, tempora, twilio, txaio, websocket-client slackclient==1.1.2 smmap2==2.0.5 # via gitdb2 From d58df0f34a0e4b9dcc9c5a9cae9038e7e0bf6e92 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Fri, 28 Feb 2020 09:50:27 -0500 Subject: [PATCH 32/41] fix sliding window calculation --- awx/main/analytics/broadcast_websocket.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/awx/main/analytics/broadcast_websocket.py b/awx/main/analytics/broadcast_websocket.py index 2b087b3dff..937c101132 100644 --- a/awx/main/analytics/broadcast_websocket.py +++ b/awx/main/analytics/broadcast_websocket.py @@ -36,16 +36,18 @@ class FixedSlidingWindow(): self.buckets = dict() self.start_time = start_time or now_seconds() - def cleanup(self, now_bucket=now_seconds()): - if self.start_time + 60 >= now_bucket: - self.start_time = now_bucket - 60 + 1 + def cleanup(self, now_bucket=None): + now_bucket = now_bucket or now_seconds() + if self.start_time + 60 <= now_bucket: + self.start_time = now_bucket + 60 + 1 # Delete old entries - for k,v in self.buckets.items(): + for k in list(self.buckets.keys()): if k < self.start_time: del self.buckets[k] - def record(self, ts=datetime.datetime.now()): + def record(self, ts=None): + ts = ts or datetime.datetime.now() now_bucket = int((ts - datetime.datetime(1970,1,1)).total_seconds()) val = self.buckets.get(now_bucket, 0) From 770b457430a013278ea902ee3ab003172d29d624 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Thu, 5 Mar 2020 14:19:46 -0500 Subject: [PATCH 33/41] redis socket support --- awx/settings/defaults.py | 4 +-- awx/settings/local_settings.py.docker_compose | 18 ------------- .../roles/image_build/files/launch_awx.sh | 1 - installer/roles/image_build/files/settings.py | 12 --------- .../templates/launch_awx_task.sh.j2 | 1 - installer/roles/kubernetes/defaults/main.yml | 3 +-- .../kubernetes/templates/configmap.yml.j2 | 10 +++++-- .../kubernetes/templates/credentials.py.j2 | 12 --------- .../kubernetes/templates/deployment.yml.j2 | 26 ++++++++++++++++++ .../roles/local_docker/defaults/main.yml | 2 -- .../roles/local_docker/tasks/compose.yml | 12 +++++++++ .../local_docker/templates/credentials.py.j2 | 12 --------- .../templates/docker-compose.yml.j2 | 6 +++++ .../local_docker/templates/environment.sh.j2 | 2 -- .../local_docker/templates/redis.conf.j2 | 4 +++ tools/docker-compose-cluster.yml | 27 ++++++++++--------- tools/docker-compose.yml | 7 ++++- tools/docker-compose/bootstrap_development.sh | 1 + tools/redis/redis.conf | 10 +++++++ tools/redis/redis_1.conf | 4 --- tools/redis/redis_2.conf | 4 --- tools/redis/redis_3.conf | 4 --- .../redis/redis_socket_ha_1/.dir_placeholder | 1 + .../redis/redis_socket_ha_2/.dir_placeholder | 1 + .../redis/redis_socket_ha_3/.dir_placeholder | 1 + .../redis_socket_standalone/.dir_placeholder | 1 + 26 files changed, 95 insertions(+), 91 deletions(-) create mode 100644 installer/roles/local_docker/templates/redis.conf.j2 create mode 100644 tools/redis/redis.conf delete mode 100644 tools/redis/redis_1.conf delete mode 100644 tools/redis/redis_2.conf delete mode 100644 tools/redis/redis_3.conf create mode 100644 tools/redis/redis_socket_ha_1/.dir_placeholder create mode 100644 tools/redis/redis_socket_ha_2/.dir_placeholder create mode 100644 tools/redis/redis_socket_ha_3/.dir_placeholder create mode 100644 tools/redis/redis_socket_standalone/.dir_placeholder diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 47d6a3522f..6989090dbc 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -421,7 +421,7 @@ os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199') BROKER_DURABILITY = True BROKER_POOL_LIMIT = None -BROKER_URL = 'redis://localhost:6379' +BROKER_URL = 'unix:///var/run/redis/redis.sock' BROKER_TRANSPORT_OPTIONS = {} CELERYBEAT_SCHEDULE = { 'tower_scheduler': { @@ -956,7 +956,7 @@ CHANNEL_LAYERS = { "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { - "hosts": [("localhost", 6379)], + "hosts": [BROKER_URL], "capacity": 10000, }, }, diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index 8d785e62ef..776b17a5de 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -12,9 +12,7 @@ # MISC PROJECT SETTINGS ############################################################################### import os -import urllib.parse import sys -from urllib import parse # Enable the following lines and install the browser extension to use Django debug toolbar # if your deployment method is not VMWare of Docker-for-Mac you may @@ -50,22 +48,6 @@ if "pytest" in sys.modules: } } -# Use Redis as the message bus for now -# Default to "just works" for single tower docker -BROKER_URL = os.environ.get('BROKER_URL', "redis://redis_1:6379") - -redis_parts = parse.urlparse(BROKER_URL) - -CHANNEL_LAYERS = { - "default": { - "BACKEND": "channels_redis.core.RedisChannelLayer", - "CONFIG": { - "hosts": [(redis_parts.hostname, redis_parts.port)], - "capacity": 10000, - }, - }, -} - # Absolute filesystem path to the directory to host projects (with playbooks). # This directory should NOT be web-accessible. PROJECTS_ROOT = '/var/lib/awx/projects/' diff --git a/installer/roles/image_build/files/launch_awx.sh b/installer/roles/image_build/files/launch_awx.sh index 360be51528..59d310276d 100755 --- a/installer/roles/image_build/files/launch_awx.sh +++ b/installer/roles/image_build/files/launch_awx.sh @@ -9,7 +9,6 @@ source /etc/tower/conf.d/environment.sh ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$MEMCACHED_HOST port=$MEMCACHED_PORT" all -ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$REDIS_HOST port=$REDIS_PORT" all ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m postgresql_db --become-user $DATABASE_USER -a "name=$DATABASE_NAME owner=$DATABASE_USER login_user=$DATABASE_USER login_host=$DATABASE_HOST login_password=$DATABASE_PASSWORD port=$DATABASE_PORT" all awx-manage collectstatic --noinput --clear diff --git a/installer/roles/image_build/files/settings.py b/installer/roles/image_build/files/settings.py index a26d71c5c9..d431e2929d 100644 --- a/installer/roles/image_build/files/settings.py +++ b/installer/roles/image_build/files/settings.py @@ -85,16 +85,4 @@ DATABASES = { if os.getenv("DATABASE_SSLMODE", False): DATABASES['default']['OPTIONS'] = {'sslmode': os.getenv("DATABASE_SSLMODE")} -BROKER_URL = 'redis://{}:{}'.format( - os.getenv("REDIS_HOST", None), - os.getenv("REDIS_PORT", "6379"),) - -CHANNEL_LAYERS = { - 'default': {'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': { - 'hosts': [(os.getenv("REDIS_HOST", None), int(os.getenv("REDIS_PORT", 6379)))] - 'capacity': 10000, - }} -} - USE_X_FORWARDED_PORT = True diff --git a/installer/roles/image_build/templates/launch_awx_task.sh.j2 b/installer/roles/image_build/templates/launch_awx_task.sh.j2 index 532f380c62..c9471a89d9 100755 --- a/installer/roles/image_build/templates/launch_awx_task.sh.j2 +++ b/installer/roles/image_build/templates/launch_awx_task.sh.j2 @@ -9,7 +9,6 @@ source /etc/tower/conf.d/environment.sh ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$MEMCACHED_HOST port=$MEMCACHED_PORT" all -ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$REDIS_HOST port=$REDIS_PORT" all ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m postgresql_db --become-user $DATABASE_USER -a "name=$DATABASE_NAME owner=$DATABASE_USER login_user=$DATABASE_USER login_host=$DATABASE_HOST login_password=$DATABASE_PASSWORD port=$DATABASE_PORT" all if [ -z "$AWX_SKIP_MIGRATIONS" ]; then diff --git a/installer/roles/kubernetes/defaults/main.yml b/installer/roles/kubernetes/defaults/main.yml index 86680f6fa0..b4bd738e60 100644 --- a/installer/roles/kubernetes/defaults/main.yml +++ b/installer/roles/kubernetes/defaults/main.yml @@ -25,8 +25,7 @@ redis_cpu_request: 500 kubernetes_redis_image: "redis" kubernetes_redis_image_tag: "latest" -kubernetes_redis_hostname: "localhost" -kubernetes_redis_port: "6379" +kubernetes_redis_config_mount_path: "/usr/local/etc/redis/redis.conf" memcached_hostname: localhost memcached_mem_request: 1 diff --git a/installer/roles/kubernetes/templates/configmap.yml.j2 b/installer/roles/kubernetes/templates/configmap.yml.j2 index 9b904e709d..9c91eebba7 100644 --- a/installer/roles/kubernetes/templates/configmap.yml.j2 +++ b/installer/roles/kubernetes/templates/configmap.yml.j2 @@ -205,5 +205,11 @@ data: USE_X_FORWARDED_PORT = True AWX_CONTAINER_GROUP_DEFAULT_IMAGE = "{{ container_groups_image }}" - BROADCAST_WEBSOCKETS_PORT = 8052 - BROADCAST_WEBSOCKETS_PROTOCOL = 'http' + BROADCAST_WEBSOCKET_PORT = 8052 + BROADCAST_WEBSOCKET_PROTOCOL = 'http' + + {{ kubernetes_deployment_name }}_redis_conf: | + unixsocket /var/run/redis/redis.sock + unixsocketperm 777 + port 0 + bind 127.0.0.1 diff --git a/installer/roles/kubernetes/templates/credentials.py.j2 b/installer/roles/kubernetes/templates/credentials.py.j2 index e76d3ad74b..84357e5414 100644 --- a/installer/roles/kubernetes/templates/credentials.py.j2 +++ b/installer/roles/kubernetes/templates/credentials.py.j2 @@ -12,15 +12,3 @@ DATABASES = { }, } } - -BROKER_URL = 'redis://{}:{}/'.format( - "{{ kubernetes_redis_hostname }}", - "{{ kubernetes_redis_port }}",) - -CHANNEL_LAYERS = { - 'default': {'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': { - 'hosts': [("{{ kubernetes_redis_hostname }}", {{ kubernetes_redis_port|int }})], - 'capacity': 10000, - }} -} diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index ec4c0a65a4..44b778f009 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -40,6 +40,7 @@ spec: service: django app: {{ kubernetes_deployment_name }} spec: + serviceAccountName: awx terminationGracePeriodSeconds: 10 {% if custom_venvs is defined %} {% set trusted_hosts = "" %} @@ -127,6 +128,9 @@ spec: subPath: SECRET_KEY readOnly: true + - name: {{ kubernetes_deployment_name }}-redis-socket + mountPath: "/var/run/redis" + resources: requests: memory: "{{ web_mem_request }}Gi" @@ -170,6 +174,9 @@ spec: mountPath: "/etc/tower/SECRET_KEY" subPath: SECRET_KEY readOnly: true + + - name: {{ kubernetes_deployment_name }}-redis-socket + mountPath: "/var/run/redis" env: - name: AWX_SKIP_MIGRATIONS value: "1" @@ -197,10 +204,19 @@ spec: - name: {{ kubernetes_deployment_name }}-redis image: {{ kubernetes_redis_image }}:{{ kubernetes_redis_image_tag }} imagePullPolicy: Always + args: ["/usr/local/etc/redis/redis.conf"] ports: - name: redis protocol: TCP containerPort: 6379 + volumeMounts: + - name: {{ kubernetes_deployment_name }}-redis-config + mountPath: "{{ kubernetes_redis_config_mount_path }}" + subPath: redis.conf + readOnly: true + + - name: {{ kubernetes_deployment_name }}-redis-socket + mountPath: "/var/run/redis" resources: requests: memory: "{{ redis_mem_request }}Gi" @@ -273,6 +289,13 @@ spec: - key: {{ kubernetes_deployment_name }}_nginx_conf path: nginx.conf + - name: {{ kubernetes_deployment_name }}-redis-config + configMap: + name: {{ kubernetes_deployment_name }}-config + items: + - key: {{ kubernetes_deployment_name }}_redis_conf + path: redis.conf + - name: "{{ kubernetes_deployment_name }}-application-credentials" secret: secretName: "{{ kubernetes_deployment_name }}-secrets" @@ -289,6 +312,9 @@ spec: - key: secret_key path: SECRET_KEY + - name: {{ kubernetes_deployment_name }}-redis-socket + emptyDir: {} + --- apiVersion: v1 kind: Service diff --git a/installer/roles/local_docker/defaults/main.yml b/installer/roles/local_docker/defaults/main.yml index 9a9277ae68..056b9ecb96 100644 --- a/installer/roles/local_docker/defaults/main.yml +++ b/installer/roles/local_docker/defaults/main.yml @@ -2,8 +2,6 @@ dockerhub_version: "{{ lookup('file', playbook_dir + '/../VERSION') }}" redis_image: "redis" -redis_hostname: "redis" -redis_port: "6379" postgresql_version: "10" postgresql_image: "postgres:{{postgresql_version}}" diff --git a/installer/roles/local_docker/tasks/compose.yml b/installer/roles/local_docker/tasks/compose.yml index 949e30c8d6..120b81cc1a 100644 --- a/installer/roles/local_docker/tasks/compose.yml +++ b/installer/roles/local_docker/tasks/compose.yml @@ -4,6 +4,12 @@ path: "{{ docker_compose_dir }}" state: directory +- name: Create Redis socket directory + file: + path: "{{ docker_compose_dir }}/redis_socket" + state: directory + mode: 0777 + - name: Create Docker Compose Configuration template: src: "{{ item }}.j2" @@ -14,8 +20,14 @@ - credentials.py - docker-compose.yml - nginx.conf + - redis.conf register: awx_compose_config +- name: Set redis config to other group readable to satisfy redis-server + file: + path: "{{ docker_compose_dir }}/redis.conf" + mode: 0666 + - name: Render SECRET_KEY file copy: content: "{{ secret_key }}" diff --git a/installer/roles/local_docker/templates/credentials.py.j2 b/installer/roles/local_docker/templates/credentials.py.j2 index 53f2df6dd1..d712636167 100644 --- a/installer/roles/local_docker/templates/credentials.py.j2 +++ b/installer/roles/local_docker/templates/credentials.py.j2 @@ -10,18 +10,6 @@ DATABASES = { } } -BROKER_URL = 'redis://{}:{}/'.format( - "{{ redis_hostname }}", - "{{ redis_port }}",) - -CHANNEL_LAYERS = { - 'default': {'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': { - 'hosts': [("{{ redis_hostname }}", {{ redis_port|int }})], - 'capacity': 10000, - }} -} - CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', diff --git a/installer/roles/local_docker/templates/docker-compose.yml.j2 b/installer/roles/local_docker/templates/docker-compose.yml.j2 index 2fd82a1005..5b5f22277a 100644 --- a/installer/roles/local_docker/templates/docker-compose.yml.j2 +++ b/installer/roles/local_docker/templates/docker-compose.yml.j2 @@ -24,6 +24,7 @@ services: - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh" - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py" - "{{ docker_compose_dir }}/nginx.conf:/etc/nginx/nginx.conf:ro" + - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw" {% if project_data_dir is defined %} - "{{ project_data_dir +':/var/lib/awx/projects:rw' }}" {% endif %} @@ -76,6 +77,7 @@ services: - "{{ docker_compose_dir }}/SECRET_KEY:/etc/tower/SECRET_KEY" - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh" - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py" + - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw" {% if project_data_dir is defined %} - "{{ project_data_dir +':/var/lib/awx/projects:rw' }}" {% endif %} @@ -119,6 +121,10 @@ services: http_proxy: {{ http_proxy | default('') }} https_proxy: {{ https_proxy | default('') }} no_proxy: {{ no_proxy | default('') }} + command: ["/usr/local/etc/redis/redis.conf"] + volumes: + - "{{ docker_compose_dir }}/redis.conf:/usr/local/etc/redis/redis.conf:ro" + - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw" memcached: image: "{{ memcached_image }}:{{ memcached_version }}" diff --git a/installer/roles/local_docker/templates/environment.sh.j2 b/installer/roles/local_docker/templates/environment.sh.j2 index 83b584b261..5053f1afbe 100644 --- a/installer/roles/local_docker/templates/environment.sh.j2 +++ b/installer/roles/local_docker/templates/environment.sh.j2 @@ -8,7 +8,5 @@ DATABASE_ADMIN_PASSWORD={{ pg_admin_password|quote }} {% endif %} MEMCACHED_HOST={{ memcached_hostname|default('memcached') }} MEMCACHED_PORT={{ memcached_port|default('11211')|quote }} -REDIS_HOST={{ redis_hostname|quote }} -REDIS_PORT={{ redis_port|quote }} AWX_ADMIN_USER={{ admin_user|quote }} AWX_ADMIN_PASSWORD={{ admin_password|quote }} diff --git a/installer/roles/local_docker/templates/redis.conf.j2 b/installer/roles/local_docker/templates/redis.conf.j2 new file mode 100644 index 0000000000..daf69de7da --- /dev/null +++ b/installer/roles/local_docker/templates/redis.conf.j2 @@ -0,0 +1,4 @@ +unixsocket /var/run/redis/redis.sock +unixsocketperm 777 +port 0 +bind 127.0.0.1 diff --git a/tools/docker-compose-cluster.yml b/tools/docker-compose-cluster.yml index 6a7f393e52..d79b89578e 100644 --- a/tools/docker-compose-cluster.yml +++ b/tools/docker-compose-cluster.yml @@ -24,7 +24,6 @@ services: #entrypoint: ["bash"] environment: CURRENT_UID: - BROKER_URL: "redis://redis_1:63791" SDB_HOST: 0.0.0.0 SDB_PORT: 5899 AWX_GROUP_QUEUES: alpha,tower @@ -32,6 +31,7 @@ services: working_dir: "/awx_devel" volumes: - "../:/awx_devel" + - "./redis/redis_socket_ha_1:/var/run/redis/" ports: - "5899-5999:5899-5999" awx-2: @@ -44,12 +44,12 @@ services: working_dir: "/awx_devel" environment: CURRENT_UID: - BROKER_URL: "redis://redis_2:63792" SDB_HOST: 0.0.0.0 SDB_PORT: 7899 AWX_GROUP_QUEUES: bravo,tower volumes: - "../:/awx_devel" + - "./redis/redis_socket_ha_2:/var/run/redis/" ports: - "7899-7999:7899-7999" awx-3: @@ -63,39 +63,42 @@ services: working_dir: "/awx_devel" environment: CURRENT_UID: - BROKER_URL: "redis://redis_3:63793" SDB_HOST: 0.0.0.0 SDB_PORT: 8899 AWX_GROUP_QUEUES: charlie,tower volumes: - "../:/awx_devel" + - "./redis/redis_socket_ha_3:/var/run/redis/" ports: - "8899-8999:8899-8999" redis_1: + user: ${CURRENT_UID} image: redis:latest - hostname: redis_1 container_name: tools_redis_1_1 - command: "redis-server /usr/local/etc/redis/redis.conf" + command: ["/usr/local/etc/redis/redis.conf"] volumes: - - "./redis/redis_1.conf:/usr/local/etc/redis/redis.conf" + - "./redis/redis.conf:/usr/local/etc/redis/redis.conf" + - "./redis/redis_socket_ha_1:/var/run/redis/" ports: - "63791:63791" redis_2: + user: ${CURRENT_UID} image: redis:latest - hostname: redis_2 container_name: tools_redis_2_1 - command: "redis-server /usr/local/etc/redis/redis.conf" + command: ["/usr/local/etc/redis/redis.conf"] volumes: - - "./redis/redis_2.conf:/usr/local/etc/redis/redis.conf" + - "./redis/redis.conf:/usr/local/etc/redis/redis.conf" + - "./redis/redis_socket_ha_2:/var/run/redis/" ports: - "63792:63792" redis_3: + user: ${CURRENT_UID} image: redis:latest - hostname: redis_3 container_name: tools_redis_3_1 - command: "redis-server /usr/local/etc/redis/redis.conf" + command: ["/usr/local/etc/redis/redis.conf"] volumes: - - "./redis/redis_3.conf:/usr/local/etc/redis/redis.conf" + - "./redis/redis.conf:/usr/local/etc/redis/redis.conf" + - "./redis/redis_socket_ha_3:/var/run/redis/" ports: - "63793:63793" postgres: diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index 353510ce84..3b9428fc27 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -32,6 +32,7 @@ services: volumes: - "../:/awx_devel" - "../awx/projects/:/var/lib/awx/projects/" + - "./redis/redis_socket_standalone:/var/run/redis/" privileged: true # A useful container that simply passes through log messages to the console # helpful for testing awx/tower logging @@ -58,4 +59,8 @@ services: container_name: tools_redis_1 ports: - "6379:6379" - + user: ${CURRENT_UID} + volumes: + - "./redis/redis.conf:/usr/local/etc/redis/redis.conf" + - "./redis/redis_socket_standalone:/var/run/redis/" + command: ["/usr/local/etc/redis/redis.conf"] diff --git a/tools/docker-compose/bootstrap_development.sh b/tools/docker-compose/bootstrap_development.sh index cd0ce68e8b..0210203949 100755 --- a/tools/docker-compose/bootstrap_development.sh +++ b/tools/docker-compose/bootstrap_development.sh @@ -4,6 +4,7 @@ set +x # Wait for the databases to come up ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=postgres port=5432" all ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=memcached port=11211" all +ansible -i "127.0.0.1," -c local -v -m wait_for -a "path=/var/run/redis/redis.sock" all # In case AWX in the container wants to connect to itself, use "docker exec" to attach to the container otherwise # TODO: FIX diff --git a/tools/redis/redis.conf b/tools/redis/redis.conf new file mode 100644 index 0000000000..ff0db4cbdf --- /dev/null +++ b/tools/redis/redis.conf @@ -0,0 +1,10 @@ +unixsocket /var/run/redis/redis.sock +unixsocketperm 770 +port 0 +# Do not actually listen to any tcp port +# but include the bind directive because without it redis will +# listen on the public interface. Port 0 causes it to NOT listen on +# the public interface. Adding the below line is an extra precaution. +# If a developer comes by later and wants to listen on a tcp port and changes +# the above port, it will ONLY listen on the local interface. +bind 127.0.0.1 diff --git a/tools/redis/redis_1.conf b/tools/redis/redis_1.conf deleted file mode 100644 index 8b6f3784d7..0000000000 --- a/tools/redis/redis_1.conf +++ /dev/null @@ -1,4 +0,0 @@ -protected-mode no -port 63791 -dir . -logfile "/tmp/redis.log" diff --git a/tools/redis/redis_2.conf b/tools/redis/redis_2.conf deleted file mode 100644 index 7f02a91f4b..0000000000 --- a/tools/redis/redis_2.conf +++ /dev/null @@ -1,4 +0,0 @@ -protected-mode no -port 63792 -dir . -logfile "/tmp/redis.log" diff --git a/tools/redis/redis_3.conf b/tools/redis/redis_3.conf deleted file mode 100644 index 203d723421..0000000000 --- a/tools/redis/redis_3.conf +++ /dev/null @@ -1,4 +0,0 @@ -protected-mode no -port 63793 -dir . -logfile "/tmp/redis.log" diff --git a/tools/redis/redis_socket_ha_1/.dir_placeholder b/tools/redis/redis_socket_ha_1/.dir_placeholder new file mode 100644 index 0000000000..7660bfed95 --- /dev/null +++ b/tools/redis/redis_socket_ha_1/.dir_placeholder @@ -0,0 +1 @@ +This dir must pre-exist and be owned by the user you are launching awx dev env as. If the dir does not exist before launching the awx dev environment then docker will create the dir and it will be owned by root. Since we start our awx dev environment with user: ${CURRENT_UID} the redis container will be unable to create a socket file in a directory owned by root. diff --git a/tools/redis/redis_socket_ha_2/.dir_placeholder b/tools/redis/redis_socket_ha_2/.dir_placeholder new file mode 100644 index 0000000000..7660bfed95 --- /dev/null +++ b/tools/redis/redis_socket_ha_2/.dir_placeholder @@ -0,0 +1 @@ +This dir must pre-exist and be owned by the user you are launching awx dev env as. If the dir does not exist before launching the awx dev environment then docker will create the dir and it will be owned by root. Since we start our awx dev environment with user: ${CURRENT_UID} the redis container will be unable to create a socket file in a directory owned by root. diff --git a/tools/redis/redis_socket_ha_3/.dir_placeholder b/tools/redis/redis_socket_ha_3/.dir_placeholder new file mode 100644 index 0000000000..7660bfed95 --- /dev/null +++ b/tools/redis/redis_socket_ha_3/.dir_placeholder @@ -0,0 +1 @@ +This dir must pre-exist and be owned by the user you are launching awx dev env as. If the dir does not exist before launching the awx dev environment then docker will create the dir and it will be owned by root. Since we start our awx dev environment with user: ${CURRENT_UID} the redis container will be unable to create a socket file in a directory owned by root. diff --git a/tools/redis/redis_socket_standalone/.dir_placeholder b/tools/redis/redis_socket_standalone/.dir_placeholder new file mode 100644 index 0000000000..7660bfed95 --- /dev/null +++ b/tools/redis/redis_socket_standalone/.dir_placeholder @@ -0,0 +1 @@ +This dir must pre-exist and be owned by the user you are launching awx dev env as. If the dir does not exist before launching the awx dev environment then docker will create the dir and it will be owned by root. Since we start our awx dev environment with user: ${CURRENT_UID} the redis container will be unable to create a socket file in a directory owned by root. From 1caa2e028759a3103a73f5985b9a286f09f2fbfc Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 2 Mar 2020 16:35:06 -0500 Subject: [PATCH 34/41] work around a limitation in postgres notify to properly support copying postgres has a limitation on its notify message size (8k), and the messages we generate for deep copying functionality easily go over this limit; instead of passing a giant nested data structure across the message bus, this change makes it so that we temporarily store the JSON structure in memcached, and look it up from *within* the task see: https://github.com/ansible/tower/issues/4162 --- awx/api/generics.py | 9 ++++++++- awx/main/tasks.py | 7 ++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/awx/api/generics.py b/awx/api/generics.py index f352019c65..e0b06f6299 100644 --- a/awx/api/generics.py +++ b/awx/api/generics.py @@ -5,10 +5,12 @@ import inspect import logging import time +import uuid import urllib.parse # Django from django.conf import settings +from django.core.cache import cache from django.db import connection from django.db.models.fields import FieldDoesNotExist from django.db.models.fields.related import OneToOneRel @@ -973,6 +975,11 @@ class CopyAPIView(GenericAPIView): if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role.members.all(): new_obj.admin_role.members.add(request.user) if sub_objs: + # store the copied object dict into memcached, because it's + # often too large for postgres' notification bus + # (which has a default maximum message size of 8k) + key = 'deep-copy-{}'.format(str(uuid.uuid4())) + cache.set(key, sub_objs, timeout=3600) permission_check_func = None if hasattr(type(self), 'deep_copy_permission_check_func'): permission_check_func = ( @@ -980,7 +987,7 @@ class CopyAPIView(GenericAPIView): ) trigger_delayed_deep_copy( self.model.__module__, self.model.__name__, - obj.pk, new_obj.pk, request.user.pk, sub_objs, + obj.pk, new_obj.pk, request.user.pk, key, permission_check_func=permission_check_func ) serializer = self._get_copy_return_serializer(new_obj) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 4ab008cd31..da7035b85f 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2856,8 +2856,13 @@ def _reconstruct_relationships(copy_mapping): @task(queue=get_local_queuename) def deep_copy_model_obj( model_module, model_name, obj_pk, new_obj_pk, - user_pk, sub_obj_list, permission_check_func=None + user_pk, uuid, permission_check_func=None ): + sub_obj_list = cache.get(uuid) + if sub_obj_list is None: + logger.error('Deep copy {} from {} to {} failed unexpectedly.'.format(model_name, obj_pk, new_obj_pk)) + return + logger.debug('Deep copy {} from {} to {}.'.format(model_name, obj_pk, new_obj_pk)) from awx.api.generics import CopyAPIView from awx.main.signals import disable_activity_stream From b58c71bb74bd5f5c836f88daa3d7806ff68cea47 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Mon, 16 Mar 2020 13:34:49 -0400 Subject: [PATCH 35/41] remove broadcast websockets view --- awx/api/urls/urls.py | 2 -- awx/api/views/metrics.py | 16 ---------------- awx/api/views/root.py | 1 - 3 files changed, 19 deletions(-) diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 4e5aa5b2f5..ac1182ddca 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -36,7 +36,6 @@ from awx.api.views import ( from awx.api.views.metrics import ( MetricsView, - BroadcastWebsocketMetricsView, ) from .organization import urls as organization_urls @@ -96,7 +95,6 @@ v2_urls = [ url(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'), url(r'^', include(oauth2_urls)), url(r'^metrics/$', MetricsView.as_view(), name='metrics_view'), - url(r'^broadcast_websocket_metrics/$', BroadcastWebsocketMetricsView.as_view(), name='broadcast_websocket_metrics_view'), url(r'^ping/$', ApiV2PingView.as_view(), name='api_v2_ping_view'), url(r'^config/$', ApiV2ConfigView.as_view(), name='api_v2_config_view'), url(r'^config/subscriptions/$', ApiV2SubscriptionView.as_view(), name='api_v2_subscription_view'), diff --git a/awx/api/views/metrics.py b/awx/api/views/metrics.py index cc6a3e3994..8d78dea21f 100644 --- a/awx/api/views/metrics.py +++ b/awx/api/views/metrics.py @@ -15,7 +15,6 @@ from rest_framework.exceptions import PermissionDenied # AWX # from awx.main.analytics import collectors from awx.main.analytics.metrics import metrics -from awx.main.analytics.broadcast_websocket import BroadcastWebsocketStatsManager from awx.api import renderers from awx.api.generics import ( @@ -41,18 +40,3 @@ class MetricsView(APIView): return Response(metrics().decode('UTF-8')) raise PermissionDenied() - -class BroadcastWebsocketMetricsView(APIView): - name = _('Broadcast Websockets') - swagger_topic = 'Broadcast Websockets' - - renderer_classes = [renderers.PlainTextRenderer, - renderers.PrometheusJSONRenderer, - renderers.BrowsableAPIRenderer,] - - def get(self, request): - ''' Show Metrics Details ''' - if (request.user.is_superuser or request.user.is_system_auditor): - stats_str = BroadcastWebsocketStatsManager.get_stats_sync() or b'' - return Response(stats_str.decode('UTF-8')) - raise PermissionDenied() diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 3ac0759530..4a15936e9b 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -107,7 +107,6 @@ class ApiVersionRootView(APIView): data['applications'] = reverse('api:o_auth2_application_list', request=request) data['tokens'] = reverse('api:o_auth2_token_list', request=request) data['metrics'] = reverse('api:metrics_view', request=request) - data['broadcast_websocket_metrics'] = reverse('api:broadcast_websocket_metrics_view', request=request) data['inventory'] = reverse('api:inventory_list', request=request) data['inventory_scripts'] = reverse('api:inventory_script_list', request=request) data['inventory_sources'] = reverse('api:inventory_source_list', request=request) From 59c9de276172b6cede0a04fe87e3079c2ecd78ee Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 17 Mar 2020 09:25:21 -0400 Subject: [PATCH 36/41] awxkit python2.7 compatible print * awxkit still supports python2.7 so do not use fancy f"" yet; instead, use .format() --- awxkit/awxkit/ws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/awxkit/ws.py b/awxkit/awxkit/ws.py index 486fd5dc4a..4136b7c278 100644 --- a/awxkit/awxkit/ws.py +++ b/awxkit/awxkit/ws.py @@ -191,7 +191,7 @@ class WSClient(object): self._pending_unsubscribe.clear() self._send(json.dumps(dict(groups={}, xrftoken=self.csrftoken))) if not self._pending_unsubscribe.wait(timeout): - raise RuntimeError(f"Failed while waiting on unsubscribe reply because timeout of {timeout} seconds was reached.") + raise RuntimeError("Failed while waiting on unsubscribe reply because timeout of {} seconds was reached.".format(timeout)) else: self._send(json.dumps(dict(groups={}, xrftoken=self.csrftoken))) From 89163f2915dc16a082bb8a60969b2e489f67184e Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 17 Mar 2020 10:08:00 -0400 Subject: [PATCH 37/41] remove redis broker url test * We use sockets everywhere. Thus, password special characters no longer are an issue. --- awx/main/tests/functional/api/test_settings.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py index 723b942c94..e96b0a72d6 100644 --- a/awx/main/tests/functional/api/test_settings.py +++ b/awx/main/tests/functional/api/test_settings.py @@ -386,14 +386,3 @@ def test_saml_x509cert_validation(patch, get, admin, headers): } }) assert resp.status_code == 200 - - -@pytest.mark.django_db -def test_broker_url_with_special_characters(): - settings.BROKER_URL = 'redis://unused:a@ns:ibl3#@redis-fancy:5672/?db=mydb' - cli = redis.from_url(settings.BROKER_URL) - assert cli.host == 'redis-fancy' - assert cli.port == 5672 - # Note: There are no usernames in redis - assert cli.password == 'a@ns:ibl3#' - assert cli.db == 'mydb' From 18f5dd6e047c65e3117730ebd5cc1066d32ab6ce Mon Sep 17 00:00:00 2001 From: chris meyers Date: Wed, 18 Mar 2020 09:02:20 -0400 Subject: [PATCH 38/41] add websocket backplane documentation --- docs/websockets.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/websockets.md b/docs/websockets.md index a1f4403ee8..25b781f637 100644 --- a/docs/websockets.md +++ b/docs/websockets.md @@ -6,7 +6,7 @@ Our channels/websocket implementation handles the communication between Tower AP Tower enlists the help of the `django-channels` library to create our communications layer. `django-channels` provides us with per-client messaging integration in our application by implementing the Asynchronous Server Gateway Interface (ASGI). -To communicate between our different services we use RabbitMQ to exchange messages. Traditionally, `django-channels` uses Redis, but Tower uses a custom `asgi_amqp` library that allows access to RabbitMQ for the same purpose. +To communicate between our different services we use websockets. Every AWX node is fully connected via a special websocket endpoint that forwards any local websocket data to all other nodes. Local websockets are backed by Redis, the channels2 default service. Inside Tower we use the `emit_channel_notification` function which places messages onto the queue. The messages are given an explicit event group and event type which we later use in our wire protocol to control message delivery to the client. @@ -50,7 +50,7 @@ When a request comes in to `nginx` and has the `Upgrade` header and is for the p `daphne` handles websocket connections proxied by nginx. -`wsbroadcast` fully connects all cluster nodes to every other cluster nodes. Sends a copy of all group websocket messages to all other cluster nodes (i.e. job event type messages). +`wsbroadcast` fully connects all cluster nodes via the `/websocket/broadcast/` endpoint to every other cluster nodes. Sends a copy of all group websocket messages to all other cluster nodes (i.e. job event type messages). ### Development - `nginx` listens on 8013/8043 instead of 80/443 From 87de0cf0b3bb8a29371e847a2e85d3864b12f8b6 Mon Sep 17 00:00:00 2001 From: chris meyers Date: Wed, 18 Mar 2020 10:55:24 -0400 Subject: [PATCH 39/41] flake8, pytest, license fixes --- awx/main/signals.py | 1 - .../tests/functional/api/test_settings.py | 1 - .../tests/functional/models/test_schedule.py | 6 +- docs/licenses/aiohttp.txt | 201 ++++++++++++++ docs/licenses/aioredis.txt | 21 ++ docs/licenses/asgi-amqp.txt | 24 -- .../{msgpack-python.txt => async-timeout.txt} | 19 +- docs/licenses/channels-redis.txt | 27 ++ docs/licenses/hiredis.txt | 1 + docs/licenses/idna-ssl.txt | 22 ++ docs/licenses/inflect.txt | 17 -- docs/licenses/jaraco.itertools.txt | 10 - docs/licenses/jsonpickle.txt | 29 -- docs/licenses/msgpack.txt | 13 + docs/licenses/multidict.txt | 201 ++++++++++++++ docs/licenses/service-identity.txt | 19 ++ docs/licenses/typing-extensions.txt | 254 ++++++++++++++++++ docs/licenses/yarl.txt | 201 ++++++++++++++ requirements/README.md | 5 - tools/docker-compose-cluster.yml | 1 - 20 files changed, 966 insertions(+), 107 deletions(-) create mode 100644 docs/licenses/aiohttp.txt create mode 100644 docs/licenses/aioredis.txt delete mode 100644 docs/licenses/asgi-amqp.txt rename docs/licenses/{msgpack-python.txt => async-timeout.txt} (94%) create mode 100644 docs/licenses/channels-redis.txt create mode 100644 docs/licenses/hiredis.txt create mode 100644 docs/licenses/idna-ssl.txt delete mode 100644 docs/licenses/inflect.txt delete mode 100644 docs/licenses/jaraco.itertools.txt delete mode 100644 docs/licenses/jsonpickle.txt create mode 100644 docs/licenses/msgpack.txt create mode 100644 docs/licenses/multidict.txt create mode 100644 docs/licenses/service-identity.txt create mode 100644 docs/licenses/typing-extensions.txt create mode 100644 docs/licenses/yarl.txt diff --git a/awx/main/signals.py b/awx/main/signals.py index 1983b335f1..a27dfb6b41 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -6,7 +6,6 @@ import contextlib import logging import threading import json -import pkg_resources import sys # Django diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py index e96b0a72d6..5c6cb16022 100644 --- a/awx/main/tests/functional/api/test_settings.py +++ b/awx/main/tests/functional/api/test_settings.py @@ -8,7 +8,6 @@ import os import time from django.conf import settings -import redis # Mock from unittest import mock diff --git a/awx/main/tests/functional/models/test_schedule.py b/awx/main/tests/functional/models/test_schedule.py index 525fdd3022..3521e9a95d 100644 --- a/awx/main/tests/functional/models/test_schedule.py +++ b/awx/main/tests/functional/models/test_schedule.py @@ -103,7 +103,8 @@ class TestComputedFields: Schedule.objects.filter(pk=s.pk).update(next_run=old_next_run) s.next_run = old_next_run prior_modified = s.modified - s.update_computed_fields() + with mock.patch('awx.main.models.schedules.emit_channel_notification'): + s.update_computed_fields() assert s.next_run != old_next_run assert s.modified == prior_modified @@ -133,7 +134,8 @@ class TestComputedFields: assert s.next_run is None assert job_template.next_schedule is None s.rrule = self.distant_rrule - s.update_computed_fields() + with mock.patch('awx.main.models.schedules.emit_channel_notification'): + s.update_computed_fields() assert s.next_run is not None assert job_template.next_schedule == s diff --git a/docs/licenses/aiohttp.txt b/docs/licenses/aiohttp.txt new file mode 100644 index 0000000000..90c9d01bc5 --- /dev/null +++ b/docs/licenses/aiohttp.txt @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2013-2020 aiohttp maintainers + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/licenses/aioredis.txt b/docs/licenses/aioredis.txt new file mode 100644 index 0000000000..35a984bbc4 --- /dev/null +++ b/docs/licenses/aioredis.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014-2017 Alexey Popravka + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docs/licenses/asgi-amqp.txt b/docs/licenses/asgi-amqp.txt deleted file mode 100644 index 56b2d91f18..0000000000 --- a/docs/licenses/asgi-amqp.txt +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2017 Ansible by Red Hat -# All Rights Reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/licenses/msgpack-python.txt b/docs/licenses/async-timeout.txt similarity index 94% rename from docs/licenses/msgpack-python.txt rename to docs/licenses/async-timeout.txt index e6c0884502..8dada3edaf 100644 --- a/docs/licenses/msgpack-python.txt +++ b/docs/licenses/async-timeout.txt @@ -1,18 +1,3 @@ -Copyright (C) 2008-2011 INADA Naoki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -193,7 +178,7 @@ Copyright (C) 2008-2011 INADA Naoki APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -201,7 +186,7 @@ Copyright (C) 2008-2011 INADA Naoki same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/docs/licenses/channels-redis.txt b/docs/licenses/channels-redis.txt new file mode 100644 index 0000000000..5f4f225dd2 --- /dev/null +++ b/docs/licenses/channels-redis.txt @@ -0,0 +1,27 @@ +Copyright (c) Django Software Foundation and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of Django nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/licenses/hiredis.txt b/docs/licenses/hiredis.txt new file mode 100644 index 0000000000..55a45a88e6 --- /dev/null +++ b/docs/licenses/hiredis.txt @@ -0,0 +1 @@ +This code is released under the BSD license, after the license of hiredis. diff --git a/docs/licenses/idna-ssl.txt b/docs/licenses/idna-ssl.txt new file mode 100644 index 0000000000..13ff0bb0c7 --- /dev/null +++ b/docs/licenses/idna-ssl.txt @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2018 aio-libs team https://github.com/aio-libs/ +Copyright (c) 2017 Ocean S. A. https://ocean.io/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/docs/licenses/inflect.txt b/docs/licenses/inflect.txt deleted file mode 100644 index 822843ed16..0000000000 --- a/docs/licenses/inflect.txt +++ /dev/null @@ -1,17 +0,0 @@ -Copyright (C) 2010 Paul Dyson - -Based upon the Perl module Lingua::EN::Inflect by Damian Conway. - -This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU Affero General Public License along with this program. If not, see . - -The original Perl module Lingua::EN::Inflect by Damian Conway is available from http://search.cpan.org/~dconway/ - -This module can be downloaded at http://pypi.python.org/pypi/inflect - -This module can be installed via easy_install inflect - -Repository available at http://github.com/pwdyson/inflect.py diff --git a/docs/licenses/jaraco.itertools.txt b/docs/licenses/jaraco.itertools.txt deleted file mode 100644 index 921ae9dd2b..0000000000 --- a/docs/licenses/jaraco.itertools.txt +++ /dev/null @@ -1,10 +0,0 @@ -# As listed on https://pypi.python.org/pypi/irc - -The MIT License (MIT) -Copyright (c) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/docs/licenses/jsonpickle.txt b/docs/licenses/jsonpickle.txt deleted file mode 100644 index 3bda2ef77d..0000000000 --- a/docs/licenses/jsonpickle.txt +++ /dev/null @@ -1,29 +0,0 @@ -Copyright (C) 2008 John Paulett (john -at- paulett.org) -Copyright (C) 2009-2016 David Aguilar (davvid -at- gmail.com) -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - 3. The name of the author may not be used to endorse or promote - products derived from this software without specific prior - written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE -GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/licenses/msgpack.txt b/docs/licenses/msgpack.txt new file mode 100644 index 0000000000..5f2280e812 --- /dev/null +++ b/docs/licenses/msgpack.txt @@ -0,0 +1,13 @@ +Copyright (C) 2008-2011 INADA Naoki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/licenses/multidict.txt b/docs/licenses/multidict.txt new file mode 100644 index 0000000000..99a9e21af0 --- /dev/null +++ b/docs/licenses/multidict.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 Andrew Svetlov + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/licenses/service-identity.txt b/docs/licenses/service-identity.txt new file mode 100644 index 0000000000..64c8a6f591 --- /dev/null +++ b/docs/licenses/service-identity.txt @@ -0,0 +1,19 @@ +Copyright (c) 2014 Hynek Schlawack + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/docs/licenses/typing-extensions.txt b/docs/licenses/typing-extensions.txt new file mode 100644 index 0000000000..583f9f6e61 --- /dev/null +++ b/docs/licenses/typing-extensions.txt @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations (now Zope +Corporation, see http://www.zope.com). In 2001, the Python Software +Foundation (PSF, see http://www.python.org/psf/) was formed, a +non-profit organization created specifically to own Python-related +Intellectual Property. Zope Corporation is a sponsoring member of +the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are +retained in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/docs/licenses/yarl.txt b/docs/licenses/yarl.txt new file mode 100644 index 0000000000..cc5cfd6790 --- /dev/null +++ b/docs/licenses/yarl.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2018, Andrew Svetlov and aio-libs team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/requirements/README.md b/requirements/README.md index 68827acdd1..ba76b65e7e 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -144,8 +144,3 @@ in the top-level Makefile. This can be removed when a solution for the external log queuing is ready. https://github.com/ansible/awx/pull/5092 - -### asgi-amqp - -This library is not compatible with channels 2 and is not expected -to become so. This drives other pins in the requirements file. diff --git a/tools/docker-compose-cluster.yml b/tools/docker-compose-cluster.yml index d79b89578e..54157de843 100644 --- a/tools/docker-compose-cluster.yml +++ b/tools/docker-compose-cluster.yml @@ -21,7 +21,6 @@ services: privileged: true image: ${DEV_DOCKER_TAG_BASE}/awx_devel:${TAG} hostname: awx-1 - #entrypoint: ["bash"] environment: CURRENT_UID: SDB_HOST: 0.0.0.0 From 7c3cbe6e58057e65f005d5c08e8bb2ca8ca28ac0 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 18 Mar 2020 12:28:57 -0400 Subject: [PATCH 40/41] add a license for redis-cli --- docs/licenses/redis.txt | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/licenses/redis.txt diff --git a/docs/licenses/redis.txt b/docs/licenses/redis.txt new file mode 100644 index 0000000000..29a3fe3845 --- /dev/null +++ b/docs/licenses/redis.txt @@ -0,0 +1,22 @@ +Copyright (c) 2012 Andy McCurdy + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. From f1ee963bd08f361e6db14b44c768dd1b30c3a9fe Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 18 Mar 2020 16:19:04 -0400 Subject: [PATCH 41/41] fix up rebased migrations --- ..._instance_ip_address.py => 0110_v370_instance_ip_address.py} | 2 +- ..._delete_channelgroup.py => 0111_v370_delete_channelgroup.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename awx/main/migrations/{0109_v370_instance_ip_address.py => 0110_v370_instance_ip_address.py} (85%) rename awx/main/migrations/{0110_delete_channelgroup.py => 0111_v370_delete_channelgroup.py} (83%) diff --git a/awx/main/migrations/0109_v370_instance_ip_address.py b/awx/main/migrations/0110_v370_instance_ip_address.py similarity index 85% rename from awx/main/migrations/0109_v370_instance_ip_address.py rename to awx/main/migrations/0110_v370_instance_ip_address.py index f3c97d5cff..914be02c52 100644 --- a/awx/main/migrations/0109_v370_instance_ip_address.py +++ b/awx/main/migrations/0110_v370_instance_ip_address.py @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('main', '0108_v370_unifiedjob_dependencies_processed'), + ('main', '0109_v370_job_template_organization_field'), ] operations = [ diff --git a/awx/main/migrations/0110_delete_channelgroup.py b/awx/main/migrations/0111_v370_delete_channelgroup.py similarity index 83% rename from awx/main/migrations/0110_delete_channelgroup.py rename to awx/main/migrations/0111_v370_delete_channelgroup.py index 7f571e587a..d17270fb90 100644 --- a/awx/main/migrations/0110_delete_channelgroup.py +++ b/awx/main/migrations/0111_v370_delete_channelgroup.py @@ -6,7 +6,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ('main', '0109_v370_instance_ip_address'), + ('main', '0110_v370_instance_ip_address'), ] operations = [