From 8faf5887754addeb4940f73a6e02a380f7623541 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Thu, 9 Nov 2017 17:17:30 -0500 Subject: [PATCH] Update package versions, settings, and tooling --- Makefile | 12 +- awx/settings/defaults.py | 139 +++++++++--------- awx/settings/development.py | 13 +- awx/settings/local_settings.py.docker_compose | 17 ++- awx/settings/production.py | 4 +- installer/image_build/files/settings.py | 12 +- .../openshift/templates/configmap.yml.j2 | 6 +- requirements/requirements.in | 25 ++-- requirements/requirements.txt | 78 +++++----- tools/docker-compose/supervisor.conf | 2 +- 10 files changed, 161 insertions(+), 147 deletions(-) diff --git a/Makefile b/Makefile index ddd8747b12..f2beb88570 100644 --- a/Makefile +++ b/Makefile @@ -203,8 +203,11 @@ develop: fi version_file: - mkdir -p /var/lib/awx/ - python -c "import awx as awx; print awx.__version__" > /var/lib/awx/.awx_version + mkdir -p /var/lib/awx/; \ + if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/awx/bin/activate; \ + fi; \ + python -c "import awx as awx; print awx.__version__" > /var/lib/awx/.awx_version; \ # Do any one-time init tasks. comma := , @@ -284,7 +287,7 @@ flower: @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/awx/bin/activate; \ fi; \ - $(PYTHON) manage.py celery flower --address=0.0.0.0 --port=5555 --broker=amqp://guest:guest@$(RABBITMQ_HOST):5672// + celery flower --address=0.0.0.0 --port=5555 --broker=amqp://guest:guest@$(RABBITMQ_HOST):5672// collectstatic: @if [ "$(VENV_BASE)" ]; then \ @@ -322,8 +325,7 @@ celeryd: @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/awx/bin/activate; \ fi; \ - $(PYTHON) manage.py celeryd -l DEBUG -B -Ofair --autoreload --autoscale=100,4 --schedule=$(CELERY_SCHEDULE_FILE) -Q tower_scheduler,tower_broadcast_all,$(COMPOSE_HOST),$(AWX_GROUP_QUEUES) -n celery@$(COMPOSE_HOST) - #$(PYTHON) manage.py celery multi show projects jobs default -l DEBUG -Q:projects projects -Q:jobs jobs -Q:default default -c:projects 1 -c:jobs 3 -c:default 3 -Ofair -B --schedule=$(CELERY_SCHEDULE_FILE) + celery worker -A awx -l DEBUG -B -Ofair --autoscale=100,4 --schedule=$(CELERY_SCHEDULE_FILE) -Q tower_scheduler,tower_broadcast_all,$(COMPOSE_HOST),$(AWX_GROUP_QUEUES) -n celery@$(COMPOSE_HOST) # Run to start the zeromq callback receiver receiver: diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index d4747a8078..4109366c0e 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -5,7 +5,6 @@ import os import re # noqa import sys import ldap -import djcelery from datetime import timedelta from kombu import Queue, Exchange @@ -42,7 +41,6 @@ def IS_TESTING(argv=None): DEBUG = True -TEMPLATE_DEBUG = DEBUG SQL_DEBUG = DEBUG ADMINS = ( @@ -195,20 +193,37 @@ CSRF_COOKIE_SECURE = True # Limit CSRF cookies to browser sessions CSRF_COOKIE_AGE = None -TEMPLATE_CONTEXT_PROCESSORS = ( # NOQA - 'django.contrib.auth.context_processors.auth', - 'django.core.context_processors.debug', - 'django.core.context_processors.i18n', - 'django.core.context_processors.media', - 'django.core.context_processors.static', - 'django.core.context_processors.tz', - 'django.contrib.messages.context_processors.messages', - 'django.core.context_processors.request', - 'awx.ui.context_processors.settings', - 'awx.ui.context_processors.version', - 'social.apps.django_app.context_processors.backends', - 'social.apps.django_app.context_processors.login_redirect', -) +TEMPLATES = [ + { + 'NAME': 'default', + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'OPTIONS': { + 'debug': DEBUG, + 'context_processors': [# NOQA + 'django.contrib.auth.context_processors.auth', + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.contrib.messages.context_processors.messages', + 'awx.ui.context_processors.settings', + 'awx.ui.context_processors.version', + 'social_django.context_processors.backends', + 'social_django.context_processors.login_redirect', + ], + 'loaders': [ + 'django.template.loaders.cached.Loader', + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + ], + }, + 'DIRS': [ + os.path.join(BASE_DIR, 'templates'), + ], + }, +] MIDDLEWARE_CLASSES = ( # NOQA 'django.contrib.sessions.middleware.SessionMiddleware', @@ -217,6 +232,7 @@ MIDDLEWARE_CLASSES = ( # NOQA 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', + 'debug_toolbar.middleware.DebugToolbarMiddleware', 'awx.main.middleware.ActivityStreamMiddleware', 'awx.sso.middleware.SocialAuthMiddleware', 'crum.CurrentRequestUserMiddleware', @@ -224,16 +240,6 @@ MIDDLEWARE_CLASSES = ( # NOQA 'awx.main.middleware.URLModificationMiddleware', ) -TEMPLATE_DIRS = ( - os.path.join(BASE_DIR, 'templates'), -) - -TEMPLATE_LOADERS = ( - ('django.template.loaders.cached.Loader', ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', - )), -) ROOT_URLCONF = 'awx.urls' @@ -248,12 +254,11 @@ INSTALLED_APPS = ( 'django.contrib.staticfiles', 'rest_framework', 'django_extensions', - 'djcelery', - 'kombu.transport.django', + 'django_celery_results', 'channels', 'polymorphic', 'taggit', - 'social.apps.django_app.default', + 'social_django', 'awx.conf', 'awx.main', 'awx.api', @@ -302,11 +307,11 @@ AUTHENTICATION_BACKENDS = ( 'awx.sso.backends.LDAPBackend', 'awx.sso.backends.RADIUSBackend', 'awx.sso.backends.TACACSPlusBackend', - 'social.backends.google.GoogleOAuth2', - 'social.backends.github.GithubOAuth2', - 'social.backends.github.GithubOrganizationOAuth2', - 'social.backends.github.GithubTeamOAuth2', - 'social.backends.azuread.AzureADOAuth2', + 'social_core.backends.google.GoogleOAuth2', + 'social_core.backends.github.GithubOAuth2', + 'social_core.backends.github.GithubOrganizationOAuth2', + 'social_core.backends.github.GithubTeamOAuth2', + 'social_core.backends.azuread.AzureADOAuth2', 'awx.sso.backends.SAMLAuth', 'django.contrib.auth.backends.ModelBackend', ) @@ -410,41 +415,35 @@ DEVSERVER_DEFAULT_PORT = '8013' # Set default ports for live server tests. os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199') -# Initialize Django-Celery. -djcelery.setup_loader() - -BROKER_URL = 'amqp://guest:guest@localhost:5672//' +CELERY_BROKER_URL = 'amqp://guest:guest@localhost:5672//' CELERY_EVENT_QUEUE_TTL = 5 -CELERY_DEFAULT_QUEUE = 'tower' +CELERY_TASK_DEFAULT_QUEUE = 'tower' CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TRACK_STARTED = True -CELERYD_TASK_TIME_LIMIT = None -CELERYD_TASK_SOFT_TIME_LIMIT = None -CELERYD_POOL_RESTARTS = True -CELERYBEAT_SCHEDULER = 'celery.beat.PersistentScheduler' -CELERYBEAT_MAX_LOOP_INTERVAL = 60 -CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend' +CELERY_TASK_TRACK_STARTED = True +CELERY_TASK_TIME_LIMIT = None +CELERY_TASK_SOFT_TIME_LIMIT = None +CELERY_WORKER_POOL_RESTARTS = True +CELERY_BEAT_SCHEDULER = 'celery.beat.PersistentScheduler' +CELERY_BEAT_MAX_LOOP_INTERVAL = 60 +CELERY_RESULT_BACKEND = 'django-db' CELERY_IMPORTS = ('awx.main.scheduler.tasks',) -CELERY_QUEUES = ( +CELERY_TASK_QUEUES = ( Queue('default', Exchange('default'), routing_key='default'), Queue('tower', Exchange('tower'), routing_key='tower'), Queue('tower_scheduler', Exchange('scheduler', type='topic'), routing_key='tower_scheduler.job.#', durable=False), Broadcast('tower_broadcast_all') ) -CELERY_ROUTES = {'awx.main.scheduler.tasks.run_task_manager': {'queue': 'tower', - 'routing_key': 'tower'}, - 'awx.main.scheduler.tasks.run_job_launch': {'queue': 'tower_scheduler', - 'routing_key': 'tower_scheduler.job.launch'}, - 'awx.main.scheduler.tasks.run_job_complete': {'queue': 'tower_scheduler', - 'routing_key': 'tower_scheduler.job.complete'}, - 'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default', - 'routing_key': 'cluster.heartbeat'}, - 'awx.main.tasks.purge_old_stdout_files': {'queue': 'default', - 'routing_key': 'cluster.heartbeat'}} +CELERY_TASK_ROUTES = { + 'awx.main.scheduler.tasks.run_task_manager': {'queue': 'tower', 'routing_key': 'tower'}, + 'awx.main.scheduler.tasks.run_job_launch': {'queue': 'tower_scheduler', 'routing_key': 'tower_scheduler.job.launch'}, + 'awx.main.scheduler.tasks.run_job_complete': {'queue': 'tower_scheduler', 'routing_key': 'tower_scheduler.job.complete'}, + 'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default', 'routing_key': 'cluster.heartbeat'}, + 'awx.main.tasks.purge_old_stdout_files': {'queue': 'default', 'routing_key': 'cluster.heartbeat'}, +} -CELERYBEAT_SCHEDULE = { +CELERY_BEAT_SCHEDULE = { 'tower_scheduler': { 'task': 'awx.main.tasks.awx_periodic_scheduler', 'schedule': timedelta(seconds=30), @@ -491,22 +490,22 @@ else: } # Social Auth configuration. -SOCIAL_AUTH_STRATEGY = 'awx.sso.strategies.django_strategy.AWXDjangoStrategy' -SOCIAL_AUTH_STORAGE = 'social.apps.django_app.default.models.DjangoStorage' +SOCIAL_AUTH_STRATEGY = 'social_django.strategy.DjangoStrategy' +SOCIAL_AUTH_STORAGE = 'social_django.models.DjangoStorage' SOCIAL_AUTH_USER_MODEL = AUTH_USER_MODEL # noqa SOCIAL_AUTH_PIPELINE = ( - 'social.pipeline.social_auth.social_details', - 'social.pipeline.social_auth.social_uid', - 'social.pipeline.social_auth.auth_allowed', - 'social.pipeline.social_auth.social_user', - 'social.pipeline.user.get_username', - 'social.pipeline.social_auth.associate_by_email', - 'social.pipeline.user.create_user', + 'social_core.pipeline.social_auth.social_details', + 'social_core.pipeline.social_auth.social_uid', + 'social_core.pipeline.social_auth.auth_allowed', + 'social_core.pipeline.social_auth.social_user', + 'social_core.pipeline.user.get_username', + 'social_core.pipeline.social_auth.associate_by_email', + 'social_core.pipeline.user.create_user', 'awx.sso.pipeline.check_user_found_or_created', - 'social.pipeline.social_auth.associate_user', - 'social.pipeline.social_auth.load_extra_data', + 'social_core.pipeline.social_auth.associate_user', + 'social_core.pipeline.social_auth.load_extra_data', 'awx.sso.pipeline.set_is_active_for_new_user', - 'social.pipeline.user.user_details', + 'social_core.pipeline.user.user_details', 'awx.sso.pipeline.prevent_inactive_login', 'awx.sso.pipeline.update_user_orgs', 'awx.sso.pipeline.update_user_teams', diff --git a/awx/settings/development.py b/awx/settings/development.py index 30bf801298..e7482520e7 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -39,13 +39,14 @@ SESSION_COOKIE_SECURE = False CSRF_COOKIE_SECURE = False # Override django.template.loaders.cached.Loader in defaults.py -TEMPLATE_LOADERS = ( +template = next((tpl_backend for tpl_backend in TEMPLATES if tpl_backend['NAME'] == 'default'), None) # noqa +template['OPTIONS']['loaders'] = ( 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ) # Disable capturing all SQL queries when running celeryd in development. -if 'celeryd' in sys.argv: +if 'celery' in sys.argv: SQL_DEBUG = False CELERYD_HIJACK_ROOT_LOGGER = False @@ -123,11 +124,11 @@ except ImportError: sys.exit(1) CLUSTER_HOST_ID = socket.gethostname() -CELERY_ROUTES['awx.main.tasks.cluster_node_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID} +CELERY_TASK_ROUTES['awx.main.tasks.cluster_node_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID} # Production only runs this schedule on controlling nodes # but development will just run it on all nodes -CELERY_ROUTES['awx.main.tasks.awx_isolated_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID} -CELERYBEAT_SCHEDULE['isolated_heartbeat'] = { +CELERY_TASK_ROUTES['awx.main.tasks.awx_isolated_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID} +CELERY_BEAT_SCHEDULE['isolated_heartbeat'] = { 'task': 'awx.main.tasks.awx_isolated_heartbeat', 'schedule': timedelta(seconds = AWX_ISOLATED_PERIODIC_CHECK), 'options': {'expires': AWX_ISOLATED_PERIODIC_CHECK * 2,} @@ -135,7 +136,7 @@ CELERYBEAT_SCHEDULE['isolated_heartbeat'] = { # Supervisor service name dictionary used for programatic restart SERVICE_NAME_DICT = { - "celery": "celeryd", + "celery": "celery", "callback": "receiver", "runworker": "channels", "uwsgi": "uwsgi", diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index 024a57ea60..96f097d0fb 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -12,6 +12,8 @@ # MISC PROJECT SETTINGS ############################################################################### import os +import urllib + def patch_broken_pipe_error(): """Monkey Patch BaseServer.handle_error to not write @@ -51,7 +53,7 @@ MANAGERS = ADMINS # Database settings to use PostgreSQL for development. DATABASES = { 'default': { - 'ENGINE': 'transaction_hooks.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'NAME': 'awx-dev', 'USER': 'awx-dev', 'PASSWORD': 'AWXsome1', @@ -67,7 +69,7 @@ DATABASES = { if is_testing(sys.argv): DATABASES = { 'default': { - 'ENGINE': 'transaction_hooks.backends.sqlite3', + 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(BASE_DIR, 'awx.sqlite3'), 'TEST': { # Test database cannot be :memory: for celery/inventory tests. @@ -79,15 +81,15 @@ if is_testing(sys.argv): MONGO_DB = 'system_tracking_test' # Celery AMQP configuration. -BROKER_URL = "amqp://{}:{}@{}/{}".format(os.environ.get("RABBITMQ_USER"), +CELERY_BROKER_URL = "amqp://{}:{}@{}/{}".format(os.environ.get("RABBITMQ_USER"), os.environ.get("RABBITMQ_PASS"), os.environ.get("RABBITMQ_HOST"), - os.environ.get("RABBITMQ_VHOST")) + urllib.quote(os.environ.get("RABBITMQ_VHOST", "/"), safe='')) CHANNEL_LAYERS = { - 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer', + 'default': {'BACKEND': 'asgi_rabbitmq.RabbitmqChannelLayer', 'ROUTING': 'awx.main.routing.channel_routing', - 'CONFIG': {'url': BROKER_URL}} + 'CONFIG': {'url': CELERY_BROKER_URL}} } # Mongo host configuration @@ -114,6 +116,7 @@ SYSTEM_UUID = '00000000-0000-0000-0000-000000000000' # timezone as the operating system. # If running in a Windows environment this must be set to the same as your # system time zone. +USE_TZ = False TIME_ZONE = None # Language code for this installation. All choices can be found here: @@ -191,7 +194,7 @@ EMAIL_SUBJECT_PREFIX = '[AWX] ' LOGGING['handlers']['syslog'] = { 'level': 'WARNING', 'filters': ['require_debug_false'], - 'class': 'django.utils.log.NullHandler', + 'class': 'logging.NullHandler', 'formatter': 'simple', } diff --git a/awx/settings/production.py b/awx/settings/production.py index 7ad7a908b2..eaec67f247 100644 --- a/awx/settings/production.py +++ b/awx/settings/production.py @@ -60,8 +60,8 @@ LOGGING['handlers']['rbac_migrations']['filename'] = '/var/log/tower/tower_rbac_ # Supervisor service name dictionary used for programatic restart SERVICE_NAME_DICT = { - "beat": "awx-celeryd-beat", - "celery": "awx-celeryd", + "beat": "awx-celery-beat", + "celery": "awx-celery", "callback": "awx-callback-receiver", "channels": "awx-channels-worker", "uwsgi": "awx-uwsgi", diff --git a/installer/image_build/files/settings.py b/installer/image_build/files/settings.py index e46051b6c5..d9a56df2f1 100644 --- a/installer/image_build/files/settings.py +++ b/installer/image_build/files/settings.py @@ -31,9 +31,9 @@ AWX_PROOT_ENABLED = False CLUSTER_HOST_ID = "awx" SYSTEM_UUID = '00000000-0000-0000-0000-000000000000' -CELERY_QUEUES += (Queue(CLUSTER_HOST_ID, Exchange(CLUSTER_HOST_ID), routing_key=CLUSTER_HOST_ID),) -CELERY_ROUTES['awx.main.tasks.cluster_node_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID} -CELERY_ROUTES['awx.main.tasks.purge_old_stdout_files'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID} +CELERY_TASK_QUEUES += (Queue(CLUSTER_HOST_ID, Exchange(CLUSTER_HOST_ID), routing_key=CLUSTER_HOST_ID),) +CELERY_TASK_ROUTES['awx.main.tasks.cluster_node_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID} +CELERY_TASK_ROUTES['awx.main.tasks.purge_old_stdout_files'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID} ############################################################################### @@ -79,7 +79,7 @@ LOGGING['handlers']['management_playbooks'] = {'class': 'logging.NullHandler'} DATABASES = { 'default': { 'ATOMIC_REQUESTS': True, - 'ENGINE': 'transaction_hooks.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'NAME': os.getenv("DATABASE_NAME", None), 'USER': os.getenv("DATABASE_USER", None), 'PASSWORD': os.getenv("DATABASE_PASSWORD", None), @@ -88,7 +88,7 @@ DATABASES = { } } -BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format( +CELERY_BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format( os.getenv("RABBITMQ_USER", None), os.getenv("RABBITMQ_PASSWORD", None), os.getenv("RABBITMQ_HOST", None), @@ -98,7 +98,7 @@ BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format( CHANNEL_LAYERS = { 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer', 'ROUTING': 'awx.main.routing.channel_routing', - 'CONFIG': {'url': BROKER_URL}} + 'CONFIG': {'url': CELERY_BROKER_URL}} } diff --git a/installer/openshift/templates/configmap.yml.j2 b/installer/openshift/templates/configmap.yml.j2 index 1cf1cc58a2..7a8f14447f 100644 --- a/installer/openshift/templates/configmap.yml.j2 +++ b/installer/openshift/templates/configmap.yml.j2 @@ -65,7 +65,7 @@ data: DATABASES = { 'default': { 'ATOMIC_REQUESTS': True, - 'ENGINE': 'transaction_hooks.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'NAME': "{{ pg_database }}", 'USER': "{{ pg_username }}", 'PASSWORD': "{{ pg_password }}", @@ -73,7 +73,7 @@ data: 'PORT': "{{ pg_port }}", } } - BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format( + CELERY_BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format( "awx", "abcdefg", "localhost", @@ -82,7 +82,7 @@ data: CHANNEL_LAYERS = { 'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer', 'ROUTING': 'awx.main.routing.channel_routing', - 'CONFIG': {'url': BROKER_URL}} + 'CONFIG': {'url': CELERY_BROKER_URL}} } CACHES = { 'default': { diff --git a/requirements/requirements.in b/requirements/requirements.in index e531173499..72b0f74066 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,28 +1,29 @@ apache-libcloud==2.0.0 appdirs==1.4.2 -asgi-amqp==0.4.1 -asgiref==1.0.1 +asgi-rabbitmq==0.5.4 +asgiref==1.1.2 azure==2.0.0rc6 backports.ssl-match-hostname==3.5.0.1 boto==2.46.1 boto3==1.4.4 -channels==0.17.3 -celery==3.1.17 -daphne>=0.15.0,<1.0.0 -Django==1.8.16 +channels==1.1.8 +celery==4.1 +daphne==1.3.0 +Django==1.11.7 django-auth-ldap==1.2.8 -django-celery==3.1.17 +django-celery-beat==1.1.0 +django-celery-results==1.0.1 django-crum==0.7.1 django-extensions==1.7.8 django-jsonfield==1.0.1 -django-polymorphic==1.2 +django-polymorphic==1.3 django-pglocks==1.0.2 django-radius==1.1.0 django-solo==1.1.2 django-split-settings==0.2.5 django-taggit==0.22.1 django-transaction-hooks==0.2 -djangorestframework==3.3.3 +djangorestframework==3.7.3 djangorestframework-yaml==1.0.3 gevent-websocket==0.9.5 irc==15.1.1 @@ -33,6 +34,7 @@ ordereddict==1.1 pexpect==4.2.1 psphere==0.5.2 psutil==5.2.2 +psycopg2==2.7.3.2 pycrypto==2.6.1 pygerduty==0.35.2 pyOpenSSL==17.0.0 @@ -41,7 +43,8 @@ python-logstash==0.4.6 python-memcached==1.58 python-radius==1.0 python-saml==2.2.1 -python-social-auth==0.2.21 +social-auth-core==1.5.0 +social-auth-app-django==2.0.0 pyvmomi==6.5 redbaron==0.6.3 requests-futures==0.9.7 @@ -50,7 +53,7 @@ shade==1.20.0 slackclient==1.0.6 tacacs_plus==1.0 twilio==6.1.0 -twisted==16.6.0 +twisted==17.9.0 uWSGI==2.0.14 xmltodict==0.11.0 pip==9.0.1 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index fbd0b0b6b6..4dfed173d8 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -5,15 +5,15 @@ # pip-compile --output-file requirements/requirements.txt requirements/requirements.in # adal==0.4.5 # via msrestazure -amqp==1.4.9 # via kombu -anyjson==0.3.3 # via kombu +amqp==2.2.2 # via kombu apache-libcloud==2.0.0 appdirs==1.4.2 -asgi-amqp==0.4.1 -asgiref==1.0.1 +asgi-rabbitmq==0.5.4 +asgiref==1.1.2 asn1crypto==0.22.0 # via cryptography -attrs==17.2.0 # via service-identity -autobahn==17.6.1 # via daphne +attrs==17.2.0 # via automat, service-identity +autobahn==17.10.1 # via daphne +automat==0.6.0 # via twisted azure-batch==1.0.0 # via azure azure-common[autorest]==1.1.4 # via azure-batch, azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-keyvault, azure-mgmt-logic, azure-mgmt-network, azure-mgmt-redis, azure-mgmt-resource, azure-mgmt-scheduler, azure-mgmt-storage, azure-servicebus, azure-servicemanagement-legacy, azure-storage azure-mgmt-batch==1.0.0 # via azure-mgmt @@ -36,47 +36,52 @@ babel==2.3.4 # via osc-lib, oslo.i18n, python-cinderclient, python- backports.functools-lru-cache==1.4 # via jaraco.functools backports.ssl-match-hostname==3.5.0.1 baron==0.6.6 # via redbaron -billiard==3.3.0.23 # via celery +billiard==3.5.0.3 # via celery boto3==1.4.4 boto==2.46.1 botocore==1.5.72 # via boto3, s3transfer -celery==3.1.17 -#certifi==2017.4.17 # via msrest +cached-property==1.3.1 # via asgi-rabbitmq +celery==4.1 +#certifi==2017.11.5 # via msrest cffi==1.10.0 # via cryptography -channels==0.17.3 +channels==1.1.8 cliff==2.7.0 # via osc-lib, python-designateclient, python-neutronclient, python-openstackclient cmd2==0.7.2 # via cliff constantly==15.1.0 # via twisted cryptography==1.9 # via adal, azure-storage, pyopenssl, secretstorage, twilio -daphne==0.15.0 +daphne==1.3.0 debtcollector==1.15.0 # via oslo.config, oslo.utils, python-designateclient, python-keystoneclient, python-neutronclient decorator==4.0.11 # via shade defusedxml==0.4.1 # via python-saml deprecation==1.0.1 # via openstacksdk django-auth-ldap==1.2.8 -django-celery==3.1.17 +django-celery-beat==1.1.0 +django-celery-results==1.0.1 django-crum==0.7.1 django-extensions==1.7.8 django-jsonfield==1.0.1 -django-polymorphic==1.2 django-pglocks==1.0.2 +django-polymorphic==1.3 django-radius==1.1.0 django-solo==1.1.2 django-split-settings==0.2.5 django-taggit==0.22.1 django-transaction-hooks==0.2 -django==1.8.16 # via channels, django-auth-ldap, django-crum, django-split-settings, django-transaction-hooks +django==1.11.7 djangorestframework-yaml==1.0.3 -djangorestframework==3.3.3 -#docutils==0.12 # via botocore +djangorestframework==3.7.3 +#dm.xmlsec.binding==1.3.2 # via python-saml +#docutils==0.14 # via botocore dogpile.cache==0.6.3 # via python-ironicclient, shade enum34==1.1.6 # via cryptography, msrest +ephem==3.7.6.0 # via django-celery-beat funcsigs==1.0.2 # via debtcollector, oslo.utils functools32==3.2.3.post2 # via jsonschema -futures==3.1.1 # via azure-storage, requests-futures, s3transfer, shade +futures==3.1.1 # via asgi-rabbitmq, azure-storage, requests-futures, s3transfer, shade gevent-websocket==0.9.5 gevent==1.2.2 # via gevent-websocket greenlet==0.4.12 # via gevent +hyperlink==17.3.1 # via twisted idna==2.5 # via cryptography, twilio incremental==17.5.0 # via twisted inflect==0.2.5 # via jaraco.itertools @@ -93,24 +98,23 @@ jaraco.stream==1.1.2 # via irc jaraco.text==1.9.2 # via irc, jaraco.collections jmespath==0.9.3 # via boto3, botocore, shade jsonpatch==1.16 # via openstacksdk, shade, warlock -jsonpickle==0.9.4 # via asgi-amqp jsonpointer==1.10 # via jsonpatch jsonschema==2.6.0 keyring==10.3.3 # via msrestazure keystoneauth1==2.21.0 # via openstacksdk, os-client-config, osc-lib, python-cinderclient, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient, shade -kombu==3.0.35 # via asgi-amqp, celery -lxml==3.8.0 # via pyvmomi +kombu==4.1.0 # via celery +lxml==3.8.0 # via dm.xmlsec.binding, pyvmomi m2crypto==0.25.1 markdown==2.6.7 monotonic==1.3 # via oslo.utils more-itertools==3.2.0 # via irc, jaraco.functools, jaraco.itertools -msgpack-python==0.4.8 # via asgi-amqp, oslo.serialization +msgpack-python==0.4.8 # via asgi-rabbitmq, oslo.serialization msrest==0.4.10 # via azure-common, msrestazure msrestazure==0.4.9 # via azure-common munch==2.1.1 # via shade netaddr==0.7.19 # via oslo.config, oslo.utils, pyrad, python-neutronclient netifaces==0.10.6 # via oslo.utils, shade -oauthlib==2.0.2 # via python-social-auth, requests-oauthlib +oauthlib==2.0.2 # via requests-oauthlib, social-auth-core openstacksdk==0.9.17 # via python-openstackclient ordereddict==1.1 os-client-config==1.27.0 # via openstacksdk, osc-lib, python-neutronclient, shade @@ -119,22 +123,22 @@ oslo.config==4.6.0 # via python-keystoneclient oslo.i18n==3.15.3 # via osc-lib, oslo.config, oslo.utils, python-cinderclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient oslo.serialization==2.18.0 # via python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient oslo.utils==3.26.0 # via osc-lib, oslo.serialization, python-cinderclient, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient -packaging==16.8 # via setuptools pbr==3.1.1 # via cliff, debtcollector, keystoneauth1, openstacksdk, osc-lib, oslo.i18n, oslo.serialization, oslo.utils, positional, python-cinderclient, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient, requestsexceptions, shade, stevedore pexpect==4.2.1 +pika==0.11.0 # via asgi-rabbitmq positional==1.1.1 # via keystoneauth1, python-keystoneclient prettytable==0.7.2 # via cliff, python-cinderclient, python-glanceclient, python-ironicclient, python-novaclient psphere==0.5.2 psutil==5.2.2 -psycopg2==2.7.1 +psycopg2==2.7.3.2 ptyprocess==0.5.1 # via pexpect pyasn1-modules==0.0.9 # via service-identity pyasn1==0.2.3 # via pyasn1-modules, service-identity pycparser==2.17 # via cffi pycrypto==2.6.1 pygerduty==0.35.2 -pyjwt==1.5.0 # via adal, python-social-auth, twilio -pyopenssl==17.0.0 # via pyvmomi, service-identity, twilio +pyjwt==1.5.0 # via adal, social-auth-core, twilio +pyopenssl==17.0.0 pyparsing==2.2.0 pyrad==2.1 # via django-radius python-cinderclient==2.2.0 # via python-openstackclient, shade @@ -148,18 +152,17 @@ python-logstash==0.4.6 python-memcached==1.58 python-neutronclient==6.3.0 # via shade python-novaclient==9.0.1 # via python-openstackclient, shade -python-openid==2.2.5 # via python-social-auth +python-openid==2.2.5 # via social-auth-core python-openstackclient==3.11.0 # via python-ironicclient python-radius==1.0 python-saml==2.2.1 -python-social-auth==0.2.21 -pytz==2017.2 # via babel, celery, irc, oslo.serialization, oslo.utils, tempora, twilio +pytz==2017.3 # via babel, celery, django, irc, oslo.serialization, oslo.utils, tempora, twilio pyvmomi==6.5 pyyaml==3.12 # via cliff, djangorestframework-yaml, os-client-config, oslo.config, psphere, python-ironicclient redbaron==0.6.3 requests-futures==0.9.7 -requests-oauthlib==0.8.0 # via msrest, python-social-auth -requests==2.14.2 # via adal, apache-libcloud, azure-servicebus, azure-servicemanagement-legacy, azure-storage, keystoneauth1, msrest, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, python-social-auth, pyvmomi, requests-futures, requests-oauthlib, slackclient, twilio +requests-oauthlib==0.8.0 # via msrest, social-auth-core +requests==2.14.2 # via adal, apache-libcloud, azure-servicebus, azure-servicemanagement-legacy, azure-storage, keystoneauth1, msrest, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-neutronclient, pyvmomi, requests-futures, requests-oauthlib, slackclient, social-auth-core, twilio requestsexceptions==1.2.0 # via os-client-config, shade rfc3986==1.0.0 # via oslo.config rply==0.7.4 # via baron @@ -168,23 +171,26 @@ secretstorage==2.3.1 # via keyring service-identity==16.0.0 shade==1.20.0 simplejson==3.11.1 # via osc-lib, python-cinderclient, python-neutronclient, python-novaclient -six==1.10.0 # via asgi-amqp, asgiref, autobahn, cliff, cmd2, cryptography, debtcollector, django-extensions, irc, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, keystoneauth1, more-itertools, munch, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, packaging, pygerduty, pyopenssl, pyrad, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, python-social-auth, pyvmomi, setuptools, shade, slackclient, stevedore, tacacs-plus, tempora, twilio, txaio, warlock, websocket-client +six==1.10.0 # via asgiref, autobahn, automat, cliff, cmd2, cryptography, debtcollector, django-extensions, irc, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, keystoneauth1, more-itertools, munch, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, pygerduty, pyopenssl, pyrad, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, pyvmomi, shade, slackclient, social-auth-app-django, social-auth-core, stevedore, tacacs-plus, tempora, twilio, txaio, warlock, websocket-client slackclient==1.0.6 +social-auth-app-django==2.0.0 +social-auth-core==1.5.0 stevedore==1.23.0 # via cliff, keystoneauth1, openstacksdk, osc-lib, oslo.config, python-designateclient, python-keystoneclient suds==0.4 # via psphere tacacs_plus==1.0 tempora==1.7 # via irc, jaraco.logging twilio==6.1.0 -twisted==16.6.0 -txaio==2.8.0 # via autobahn -typing==3.6.1 # via m2crypto +twisted==17.9.0 +txaio==2.8.2 # via autobahn +typing==3.6.2 # via m2crypto unicodecsv==0.14.1 # via cliff uwsgi==2.0.14 +vine==1.1.4 # via amqp warlock==1.2.0 # via python-glanceclient websocket-client==0.44.0 # via slackclient wrapt==1.10.10 # via debtcollector, positional, python-glanceclient xmltodict==0.11.0 -zope.interface==4.4.2 # via twisted +zope.interface==4.4.3 # via twisted # The following packages are considered to be unsafe in a requirements file: pip==9.0.1 diff --git a/tools/docker-compose/supervisor.conf b/tools/docker-compose/supervisor.conf index bf95d4e477..30c1fbba46 100644 --- a/tools/docker-compose/supervisor.conf +++ b/tools/docker-compose/supervisor.conf @@ -4,7 +4,7 @@ minfds = 4096 nodaemon=true [program:celeryd] -command = python manage.py celeryd -l DEBUG -B --autoreload --autoscale=20,3 --schedule=/celerybeat-schedule -Q tower_scheduler,tower_broadcast_all,%(ENV_AWX_GROUP_QUEUES)s,%(ENV_HOSTNAME)s -n celery@%(ENV_HOSTNAME)s +command = celery -l DEBUG -B --autoreload --autoscale=20,3 --schedule=/celerybeat-schedule -Q tower_scheduler,tower_broadcast_all,%(ENV_AWX_GROUP_QUEUES)s,%(ENV_HOSTNAME)s -n celery@%(ENV_HOSTNAME)s autostart = true autorestart = true redirect_stderr=true