From 20b5aa7424d2725470d3ff168202d1e616f36e3d Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 7 Oct 2016 09:58:47 -0400 Subject: [PATCH 01/21] Fix some issues with management commands for clustering --- .../management/commands/deprovision_node.py | 27 +++++++++++++++++++ .../management/commands/register_instance.py | 17 ++++++------ 2 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 awx/main/management/commands/deprovision_node.py diff --git a/awx/main/management/commands/deprovision_node.py b/awx/main/management/commands/deprovision_node.py new file mode 100644 index 0000000000..ac27960daa --- /dev/null +++ b/awx/main/management/commands/deprovision_node.py @@ -0,0 +1,27 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved + +from django.core.management.base import CommandError, BaseCommand +from optparse import make_option +from awx.main.models import Instance + + +class Command(BaseCommand): + """ + Deprovision a Tower cluster node + """ + + option_list = BaseCommand.option_list + ( + make_option('--name', dest='name', type='string', + help='Hostname used during provisioning'), + ) + + def handle(self, **options): + # Get the instance. + instance = Instance.objects.filter(hostname=options.get('name')) + if instance.exists(): + instance.delete() + print('Successfully removed') + else: + print('No instance found matching name {}'.format(options.get('name'))) + diff --git a/awx/main/management/commands/register_instance.py b/awx/main/management/commands/register_instance.py index 3355dcf983..04b2b08a8a 100644 --- a/awx/main/management/commands/register_instance.py +++ b/awx/main/management/commands/register_instance.py @@ -4,27 +4,26 @@ from awx.main.models import Instance from django.conf import settings -from django.core.management.base import CommandError, NoArgsCommand +from optparse import make_option +from django.core.management.base import CommandError, BaseCommand -class Command(NoArgsCommand): +class Command(BaseCommand): """ Internal tower command. Regsiter this instance with the database for HA tracking. """ - option_list = NoArgsCommand.option_list + ( + option_list = BaseCommand.option_list + ( make_option('--hostname', dest='hostname', type='string', - help='Hostname used during provisioning') + help='Hostname used during provisioning'), ) - def handle(self, *args, **options): - super(Command, self).handle(**options) + def handle(self, **options): uuid = settings.SYSTEM_UUID - instance = Instance.objects.filter(hostname=options.get('hostname')) if instance.exists(): - print("Instance already registered %s" % instance_str(instance[0])) + print("Instance already registered {}".format(instance[0])) return instance = Instance(uuid=uuid, hostname=options.get('hostname')) instance.save() - print('Successfully registered instance %s.' % instance_str(instance)) + print('Successfully registered instance {}'.format(instance)) From 2a844b9c426b0de6794cb14eb0c2cb1e99ffd1ac Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 7 Oct 2016 10:55:06 -0400 Subject: [PATCH 02/21] Add api node request servicer to response headers --- awx/api/generics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/api/generics.py b/awx/api/generics.py index 8e6471c1c7..4c4247b23d 100644 --- a/awx/api/generics.py +++ b/awx/api/generics.py @@ -104,6 +104,7 @@ class APIView(views.APIView): logger.warn(status_msg) response = super(APIView, self).finalize_response(request, response, *args, **kwargs) time_started = getattr(self, 'time_started', None) + response['X-API-Node'] = settings.CLUSTER_HOST_ID if time_started: time_elapsed = time.time() - self.time_started response['X-API-Time'] = '%0.3fs' % time_elapsed From 0992e354e3b9e9f3fe338086bbec3a022f080b8c Mon Sep 17 00:00:00 2001 From: Chris Church Date: Fri, 7 Oct 2016 14:13:51 -0400 Subject: [PATCH 03/21] Prevent removing license via PUT/PATCH/DELETE to /api/v1/settings/system/. --- awx/conf/views.py | 4 ++- awx/main/tests/conftest.py | 13 ++++++++ .../tests/functional/api/test_settings.py | 31 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 awx/main/tests/functional/api/test_settings.py diff --git a/awx/conf/views.py b/awx/conf/views.py index 5dfa71b84d..f6af0329d2 100644 --- a/awx/conf/views.py +++ b/awx/conf/views.py @@ -97,6 +97,8 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView): settings_qs = self.get_queryset() user = self.request.user if self.category_slug == 'user' else None for key, value in serializer.validated_data.items(): + if key == 'LICENSE': + continue setattr(serializer.instance, key, value) # Always encode "raw" strings as JSON. if isinstance(value, basestring): @@ -114,7 +116,7 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView): return Response(status=status.HTTP_204_NO_CONTENT) def perform_destroy(self, instance): - for setting in self.get_queryset(): + for setting in self.get_queryset().exclude(key='LICENSE'): setting.delete() diff --git a/awx/main/tests/conftest.py b/awx/main/tests/conftest.py index 9b2b00455c..3412ca1ed8 100644 --- a/awx/main/tests/conftest.py +++ b/awx/main/tests/conftest.py @@ -1,5 +1,6 @@ # Python +import time import pytest from awx.main.tests.factories import ( @@ -52,3 +53,15 @@ def get_ssh_version(mocker): @pytest.fixture def job_template_with_survey_passwords_unit(job_template_with_survey_passwords_factory): return job_template_with_survey_passwords_factory(persisted=False) + +@pytest.fixture +def enterprise_license(): + from awx.main.task_engine import TaskEnhancer + return TaskEnhancer( + company_name='AWX', + contact_name='AWX Admin', + contact_email='awx@example.com', + license_date=int(time.time() + 3600), + instance_count=10000, + license_type='enterprise', + ).enhance() diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py new file mode 100644 index 0000000000..51314defb5 --- /dev/null +++ b/awx/main/tests/functional/api/test_settings.py @@ -0,0 +1,31 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +# Python +import pytest +import mock + +# Django +from django.core.urlresolvers import reverse + +# AWX +from awx.conf.models import Setting + + +@pytest.mark.django_db +def test_license_cannot_be_removed_via_system_settings(get, put, patch, delete, admin, enterprise_license): + url = reverse('api:setting_singleton_detail', args=('system',)) + response = get(url, user=admin, expect=200) + assert not response.data['LICENSE'] + Setting.objects.create(key='LICENSE', value=enterprise_license) + response = get(url, user=admin, expect=200) + assert response.data['LICENSE'] + put(url, user=admin, data=response.data, expect=200) + response = get(url, user=admin, expect=200) + assert response.data['LICENSE'] + patch(url, user=admin, data={}, expect=200) + response = get(url, user=admin, expect=200) + assert response.data['LICENSE'] + delete(url, user=admin, expect=204) + response = get(url, user=admin, expect=200) + assert response.data['LICENSE'] From 47b2b4322b639d7fac24bacf744daa2c341a2313 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Mon, 10 Oct 2016 10:25:53 -0400 Subject: [PATCH 04/21] use created and modified dates from TowerSettings when migrating to conf Settings --- awx/conf/migrations/0002_v310_copy_tower_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/conf/migrations/0002_v310_copy_tower_settings.py b/awx/conf/migrations/0002_v310_copy_tower_settings.py index 7b1422ba97..2ab255debb 100644 --- a/awx/conf/migrations/0002_v310_copy_tower_settings.py +++ b/awx/conf/migrations/0002_v310_copy_tower_settings.py @@ -22,6 +22,8 @@ def copy_tower_settings(apps, schema_editor): setting, created = Setting.objects.get_or_create( key=tower_setting.key, user=tower_setting.user, + created=tower_setting.created, + modified=tower_setting.modified, defaults=dict(value=value), ) if not created and setting.value != value: From d57c0152fba93f35104064871b7777326144b4aa Mon Sep 17 00:00:00 2001 From: Ryan Fitzpatrick Date: Mon, 10 Oct 2016 13:32:34 -0400 Subject: [PATCH 05/21] Include defaul channel layer route and upgrade amqp version (#3675) --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index c2effa3802..ce9bf8b6c4 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,5 @@ git+https://github.com/chrismeyersfsu/ansiconv.git@tower_1.0.0#egg=ansiconv -amqp==1.4.5 +amqp==1.4.9 anyjson==0.3.3 appdirs==1.4.0 azure==2.0.0rc2 From 21a04f196c728630d8b3f4319f466600bc15ff4b Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 10 Oct 2016 17:13:02 -0400 Subject: [PATCH 06/21] workflow serializer fixes --- awx/api/serializers.py | 7 +++++-- awx/main/access.py | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index b000114b94..9be27357f4 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2232,11 +2232,14 @@ class WorkflowNodeBaseSerializer(BaseSerializer): job_tags = serializers.SerializerMethodField() limit = serializers.SerializerMethodField() skip_tags = serializers.SerializerMethodField() + success_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True) + failure_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True) + always_nodes = serializers.PrimaryKeyRelatedField(many=True, read_only=True) class Meta: - fields = ('id', 'url', 'related', 'unified_job_template', + fields = ('*', '-name', '-description', 'id', 'url', 'related', + 'unified_job_template', 'success_nodes', 'failure_nodes', 'always_nodes', 'inventory', 'credential', 'job_type', 'job_tags', 'skip_tags', 'limit', 'skip_tags') - read_only_fields = ('success_nodes', 'failure_nodes', 'always_nodes') def get_related(self, obj): res = super(WorkflowNodeBaseSerializer, self).get_related(obj) diff --git a/awx/main/access.py b/awx/main/access.py index c6285beb5a..46de40b2e2 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1273,6 +1273,7 @@ class WorkflowJobTemplateNodeAccess(BaseAccess): qs = self.model.objects.filter( workflow_job_template__in=WorkflowJobTemplate.accessible_objects( self.user, 'read_role')) + qs = qs.prefetch_related('success_nodes', 'failure_nodes', 'always_nodes') return qs def can_use_prompted_resources(self, data): @@ -1367,6 +1368,8 @@ class WorkflowJobNodeAccess(BaseAccess): qs = self.model.objects.filter( workflow_job__workflow_job_template__in=WorkflowJobTemplate.accessible_objects( self.user, 'read_role')) + qs = qs.select_related('unified_job_template', 'job') + qs = qs.prefetch_related('success_nodes', 'failure_nodes', 'always_nodes') return qs def can_add(self, data): From 27d29ce2046721bbef478479cdc6501a5db2e86f Mon Sep 17 00:00:00 2001 From: Graham Mainwaring Date: Tue, 11 Oct 2016 12:30:06 -0400 Subject: [PATCH 07/21] Include Powershell scripts for Windows scan jobs --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index b52764c7e8..2ad81b781a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,6 +6,7 @@ recursive-include awx/ui/templates *.html recursive-include awx/ui/static * recursive-include awx/playbooks *.yml recursive-include awx/lib/site-packages * +recursive-include awx/plugins *.ps1 recursive-include requirements *.txt recursive-include config * recursive-include docs/licenses * From 9274b90b9b54bce6cba326277c2aaac80bbdda0c Mon Sep 17 00:00:00 2001 From: jaredevantabor Date: Tue, 11 Oct 2016 14:22:42 -0700 Subject: [PATCH 08/21] switching websocket to secure websocket (wss://) dpending on protocol --- .../client/src/shared/socket/socket.service.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/awx/ui/client/src/shared/socket/socket.service.js b/awx/ui/client/src/shared/socket/socket.service.js index 6a992d4e5f..056bd09bbf 100644 --- a/awx/ui/client/src/shared/socket/socket.service.js +++ b/awx/ui/client/src/shared/socket/socket.service.js @@ -13,7 +13,17 @@ export default init: function() { var self = this, host = window.location.host, - url = "ws://" + host + "/websocket/"; + protocol, + url; + + if($location.protocol() === 'http'){ + protocol = 'ws'; + } + if($location.protocol() === 'https'){ + protocol = 'wss'; + } + url = `${protocol}://${host}/websocket/`; + if (!$rootScope.sessionTimer || ($rootScope.sessionTimer && !$rootScope.sessionTimer.isExpired())) { // We have a valid session token, so attempt socket connection $log.debug('Socket connecting to: ' + url); @@ -83,7 +93,7 @@ export default // The naming scheme is "ws" then a // dash (-) and the group_name, then the job ID // ex: 'ws-jobs-' - str = `ws-${data.group_name}-${data.job}` + str = `ws-${data.group_name}-${data.job}`; } else if(data.group_name==="ad_hoc_command_events"){ // The naming scheme is "ws" then a @@ -194,7 +204,7 @@ export default // This function is used for add a state resolve to all states, // socket-enabled AND socket-disabled, and whether the $state // requires a subscribe or an unsubscribe - self = this; + var self = this; socketPromise.promise.then(function(){ if(!state.socket){ state.socket = {groups: {}}; From 2488e1e3f0ef5ccef5bcfe288d73058f2a6d6d86 Mon Sep 17 00:00:00 2001 From: Chris Church Date: Wed, 12 Oct 2016 13:40:05 -0400 Subject: [PATCH 09/21] Show actual settings module in use. --- awx/conf/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/awx/conf/settings.py b/awx/conf/settings.py index 1e156635c0..81159c80a9 100644 --- a/awx/conf/settings.py +++ b/awx/conf/settings.py @@ -191,6 +191,10 @@ class SettingsWrapper(UserSettingsHolder): def _get_default(self, name): return getattr(self.default_settings, name) + @property + def SETTINGS_MODULE(self): + return self._get_default('SETTINGS_MODULE') + def __getattr__(self, name): value = empty if name in self._get_supported_settings(): From 3cfe544ada01bc9b12753a692c21fad046433920 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 12 Oct 2016 13:54:13 -0400 Subject: [PATCH 10/21] update workflows to work with new channels implementation --- awx/main/models/workflow.py | 2 +- awx/main/tasks.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 91983ca673..62224b9509 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -384,6 +384,6 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, JobNotificationMixin, Workflow if res: self.status = 'running' self.save() - self.socketio_emit_status("running") + self.websocket_emit_status("running") return res diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 05a172b6da..22129135fb 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1685,7 +1685,7 @@ class RunWorkflowJob(BaseTask): def run(self, pk, **kwargs): #Run the job/task and capture its output. instance = self.update_model(pk, status='running', celery_task_id=self.request.id) - instance.socketio_emit_status("running") + instance.websocket_emit_status("running") # FIXME: Currently, the workflow job busy waits until the graph run is # complete. Instead, the workflow job should return or never even run, @@ -1707,6 +1707,6 @@ class RunWorkflowJob(BaseTask): instance = self.update_model(instance.pk, status='successful') break time.sleep(1) - instance.socketio_emit_status(instance.status) + instance.websocket_emit_status(instance.status) # TODO: Handle cancel ''' From 8883738a7c700c25dd0a6b2e1ae43db5748dcbc4 Mon Sep 17 00:00:00 2001 From: Chris Church Date: Wed, 12 Oct 2016 14:47:22 -0400 Subject: [PATCH 11/21] Fix issue when string list settings field is null. Resolves #3683. --- awx/conf/fields.py | 6 ++++++ awx/settings/defaults.py | 3 --- awx/sso/conf.py | 6 ++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/awx/conf/fields.py b/awx/conf/fields.py index ae299137e6..bd2e047ae7 100644 --- a/awx/conf/fields.py +++ b/awx/conf/fields.py @@ -20,8 +20,14 @@ logger = logging.getLogger('awx.conf.fields') class StringListField(ListField): + child = CharField() + def to_representation(self, value): + if value is None and self.allow_null: + return None + return super(StringListField, self).to_representation(value) + class URLField(CharField): diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index ecc27fcbf0..c740bed39b 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -473,9 +473,6 @@ SOCIAL_AUTH_SAML_TECHNICAL_CONTACT = {} SOCIAL_AUTH_SAML_SUPPORT_CONTACT = {} SOCIAL_AUTH_SAML_ENABLED_IDPS = {} -SOCIAL_AUTH_ORGANIZATION_MAP = {} -SOCIAL_AUTH_TEAM_MAP = {} - # Any ANSIBLE_* settings will be passed to the subprocess environment by the # celery task. diff --git a/awx/sso/conf.py b/awx/sso/conf.py index 5bcff670cf..45f75b35af 100644 --- a/awx/sso/conf.py +++ b/awx/sso/conf.py @@ -126,7 +126,8 @@ register( register( 'SOCIAL_AUTH_ORGANIZATION_MAP', field_class=fields.SocialOrganizationMapField, - default={}, + allow_null=True, + default=None, label=_('Social Auth Organization Map'), help_text=SOCIAL_AUTH_ORGANIZATION_MAP_HELP_TEXT, category=_('Authentication'), @@ -137,7 +138,8 @@ register( register( 'SOCIAL_AUTH_TEAM_MAP', field_class=fields.SocialTeamMapField, - default={}, + allow_null=True, + default=None, label=_('Social Auth Team Map'), help_text=SOCIAL_AUTH_TEAM_MAP_HELP_TEXT, category=_('Authentication'), From c80a8a9c1431ac7718d8ec553e48aeb618cc6d61 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Wed, 12 Oct 2016 16:07:25 -0400 Subject: [PATCH 12/21] Exclude test directory from setup bundle tarballs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also using the rsync strategy in the tar-build target. This doesn’t leave behind the empty setup/test directory. --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 3d8ffc7e2c..bb4d3021c0 100644 --- a/Makefile +++ b/Makefile @@ -519,10 +519,10 @@ release_build: # Build setup tarball tar-build/$(SETUP_TAR_FILE): @mkdir -p tar-build - @cp -a setup tar-build/$(SETUP_TAR_NAME) + @rsync -az --exclude /test setup/ tar-build/$(SETUP_TAR_NAME) @rsync -az docs/licenses tar-build/$(SETUP_TAR_NAME)/ @cd tar-build/$(SETUP_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all - @cd tar-build && tar -czf $(SETUP_TAR_FILE) --exclude "*/all.in" --exclude "**/test/*" $(SETUP_TAR_NAME)/ + @cd tar-build && tar -czf $(SETUP_TAR_FILE) --exclude "*/all.in" $(SETUP_TAR_NAME)/ @ln -sf $(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_LINK) tar-build/$(SETUP_TAR_CHECKSUM): @@ -559,7 +559,7 @@ setup-bundle-build: # TODO - Somehow share implementation with setup_tarball setup-bundle-build/$(OFFLINE_TAR_FILE): - cp -a setup setup-bundle-build/$(OFFLINE_TAR_NAME) + rsync -az --exclude /test setup/ setup-bundle-build/$(OFFLINE_TAR_NAME) rsync -az docs/licenses setup-bundle-build/$(OFFLINE_TAR_NAME)/ cd setup-bundle-build/$(OFFLINE_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all $(PYTHON) $(DEPS_SCRIPT) -d $(DIST) -r $(DIST_MAJOR) -u $(AW_REPO_URL) -s setup-bundle-build/$(OFFLINE_TAR_NAME) -v -v -v From 5d4cf9d4fc18cb658a3dbfcf35c4652e0752ce90 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 15 Sep 2016 15:48:18 -0400 Subject: [PATCH 13/21] Add job artifacts and workflow artifact passing artifacts redact from job when no_log is set parent no_log artifacts treated as survey passwords --- awx/api/serializers.py | 8 ++- .../commands/run_callback_receiver.py | 19 ++++++ awx/main/migrations/0040_v310_artifacts.py | 25 ++++++++ awx/main/models/jobs.py | 14 +++++ awx/main/models/unified_jobs.py | 7 +-- awx/main/models/workflow.py | 52 +++++++++++++--- awx/main/tasks.py | 7 ++- .../tests/functional/models/test_workflow.py | 62 ++++++++++++++++++- .../tests/unit/models/test_workflow_unit.py | 2 + awx/plugins/callback/job_event_callback.py | 6 ++ awx/plugins/library/set_artifact.py | 59 ++++++++++++++++++ 11 files changed, 243 insertions(+), 18 deletions(-) create mode 100644 awx/main/migrations/0040_v310_artifacts.py create mode 100644 awx/plugins/library/set_artifact.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 9be27357f4..ded5d454c3 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1920,13 +1920,14 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer): ask_job_type_on_launch = serializers.ReadOnlyField() ask_inventory_on_launch = serializers.ReadOnlyField() ask_credential_on_launch = serializers.ReadOnlyField() + artifacts = serializers.SerializerMethodField() class Meta: model = Job fields = ('*', 'job_template', 'passwords_needed_to_start', 'ask_variables_on_launch', 'ask_limit_on_launch', 'ask_tags_on_launch', 'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_inventory_on_launch', 'ask_credential_on_launch', - 'allow_simultaneous',) + 'allow_simultaneous', 'artifacts',) def get_related(self, obj): res = super(JobSerializer, self).get_related(obj) @@ -1949,6 +1950,11 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer): res['relaunch'] = reverse('api:job_relaunch', args=(obj.pk,)) return res + def get_artifacts(self, obj): + if obj: + return obj.display_artifacts() + return {} + def to_internal_value(self, data): # When creating a new job and a job template is specified, populate any # fields not provided in data from the job template. diff --git a/awx/main/management/commands/run_callback_receiver.py b/awx/main/management/commands/run_callback_receiver.py index d2b89cd44d..4f777cd40e 100644 --- a/awx/main/management/commands/run_callback_receiver.py +++ b/awx/main/management/commands/run_callback_receiver.py @@ -4,6 +4,7 @@ # Python import datetime import logging +import json from kombu import Connection, Exchange, Queue from kombu.mixins import ConsumerMixin @@ -80,6 +81,7 @@ class CallbackBrokerWorker(ConsumerMixin): event_uuid = payload.get("uuid", '') parent_event_uuid = payload.get("parent_uuid", '') + artifact_data = payload.get("artifact_data", None) # Sanity check: Don't honor keys that we don't recognize. for key in payload.keys(): @@ -123,6 +125,23 @@ class CallbackBrokerWorker(ConsumerMixin): except DatabaseError as e: logger.error("Database Error Saving Job Event: {}".format(e)) + if artifact_data: + try: + self.process_artifacts(artifact_data, res, payload) + except DatabaseError as e: + logger.error("Database Error Saving Job Artifacts: {}".format(e)) + + def process_artifacts(self, artifact_data, res, payload): + artifact_dict = json.loads(artifact_data) + if res and isinstance(res, dict): + if res.get('_ansible_no_log', False): + artifact_dict['_ansible_no_log'] = True + if artifact_data is not None: + parent_job = Job.objects.filter(pk=payload['job_id']).first() + if parent_job is not None and parent_job.artifacts != artifact_dict: + parent_job.artifacts = artifact_dict + parent_job.save(update_fields=['artifacts']) + class Command(NoArgsCommand): ''' diff --git a/awx/main/migrations/0040_v310_artifacts.py b/awx/main/migrations/0040_v310_artifacts.py new file mode 100644 index 0000000000..af1c66f485 --- /dev/null +++ b/awx/main/migrations/0040_v310_artifacts.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0039_v310_channelgroup'), + ] + + operations = [ + migrations.AddField( + model_name='job', + name='artifacts', + field=jsonfield.fields.JSONField(default={}, editable=False, blank=True), + ), + migrations.AddField( + model_name='workflowjobnode', + name='ancestor_artifacts', + field=jsonfield.fields.JSONField(default={}, editable=False, blank=True), + ), + ] diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index e146a266be..27efff9cc7 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -550,6 +550,11 @@ class Job(UnifiedJob, JobOptions, JobNotificationMixin): default={}, editable=False, ) + artifacts = JSONField( + blank=True, + default={}, + editable=False, + ) @classmethod def _get_parent_field_name(cls): @@ -775,6 +780,15 @@ class Job(UnifiedJob, JobOptions, JobNotificationMixin): else: return self.extra_vars + def display_artifacts(self): + ''' + Hides artifacts if they are marked as no_log type artifacts. + ''' + artifacts = self.artifacts + if artifacts.get('_ansible_no_log', False): + return "$hidden due to Ansible no_log flag$" + return artifacts + def _survey_search_and_replace(self, content): # Use job template survey spec to identify password fields. # Then lookup password fields in extra_vars and save the values diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 3a30906e02..d40f50cca3 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -348,11 +348,10 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio create_kwargs[field_name] = getattr(self, field_name) new_kwargs = self._update_unified_job_kwargs(**create_kwargs) unified_job = unified_job_class(**new_kwargs) - # For JobTemplate-based jobs with surveys, save list for perma-redaction - if (hasattr(self, 'survey_spec') and getattr(self, 'survey_enabled', False) and - not getattr(unified_job, 'survey_passwords', False)): + # For JobTemplate-based jobs with surveys, add passwords to list for perma-redaction + if hasattr(self, 'survey_spec') and getattr(self, 'survey_enabled', False): password_list = self.survey_password_variables() - hide_password_dict = {} + hide_password_dict = getattr(unified_job, 'survey_passwords', {}) for password in password_list: hide_password_dict[password] = REPLACE_STR unified_job.survey_passwords = hide_password_dict diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 91983ca673..3c175be064 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -21,7 +21,9 @@ from awx.main.models.rbac import ( ) from awx.main.fields import ImplicitRoleField from awx.main.models.mixins import ResourceMixin +from awx.main.redact import REPLACE_STR +from copy import copy import yaml import json @@ -124,6 +126,13 @@ class WorkflowNodeBase(CreatedModifiedModel): data['missing'] = missing_dict return data + def get_parent_nodes(self): + '''Returns queryset containing all parents of this node''' + success_parents = getattr(self, '%ss_success' % self.__class__.__name__.lower()).all() + failure_parents = getattr(self, '%ss_failure' % self.__class__.__name__.lower()).all() + always_parents = getattr(self, '%ss_always' % self.__class__.__name__.lower()).all() + return success_parents | failure_parents | always_parents + @classmethod def _get_workflow_job_field_names(cls): ''' @@ -175,11 +184,22 @@ class WorkflowJobNode(WorkflowNodeBase): default=None, on_delete=models.CASCADE, ) + ancestor_artifacts = JSONField( + blank=True, + default={}, + editable=False, + ) def get_absolute_url(self): return reverse('api:workflow_job_node_detail', args=(self.pk,)) def get_job_kwargs(self): + ''' + In advance of creating a new unified job as part of a workflow, + this method builds the attributes to use + It alters the node by saving its updated version of + ancestor_artifacts, making it available to subsequent nodes. + ''' # reject/accept prompted fields data = {} ujt_obj = self.unified_job_template @@ -189,19 +209,31 @@ class WorkflowJobNode(WorkflowNodeBase): accepted_fields.pop(fd) data.update(accepted_fields) # TODO: decide what to do in the event of missing fields + # build ancestor artifacts, save them to node model for later + aa_dict = {} + for parent_node in self.get_parent_nodes(): + aa_dict.update(parent_node.ancestor_artifacts) + if parent_node.job and hasattr(parent_node.job, 'artifacts'): + aa_dict.update(parent_node.job.artifacts) + if aa_dict: + self.ancestor_artifacts = aa_dict + self.save(update_fields=['ancestor_artifacts']) + if '_ansible_no_log' in aa_dict: + # TODO: merge Workflow Job survey passwords into this + password_dict = {} + for key in aa_dict: + if key != '_ansible_no_log': + password_dict[key] = REPLACE_STR + data['survey_passwords'] = password_dict # process extra_vars + # TODO: still lack consensus about variable precedence extra_vars = {} if self.workflow_job and self.workflow_job.extra_vars: - try: - WJ_json_extra_vars = json.loads( - (self.workflow_job.extra_vars or '').strip() or '{}') - except ValueError: - try: - WJ_json_extra_vars = yaml.safe_load(self.workflow_job.extra_vars) - except yaml.YAMLError: - WJ_json_extra_vars = {} - extra_vars.update(WJ_json_extra_vars) - # TODO: merge artifacts, add ancestor_artifacts to kwargs + extra_vars.update(self.workflow_job.extra_vars_dict) + if aa_dict: + functional_aa_dict = copy(aa_dict) + functional_aa_dict.pop('_ansible_no_log', None) + extra_vars.update(functional_aa_dict) if extra_vars: data['extra_vars'] = extra_vars return data diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 05a172b6da..2c1dc45538 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -821,6 +821,11 @@ class RunJob(BaseTask): env['ANSIBLE_CACHE_PLUGINS'] = self.get_path_to('..', 'plugins', 'fact_caching') env['ANSIBLE_CACHE_PLUGIN'] = "tower" env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] = "tcp://127.0.0.1:%s" % str(settings.FACT_CACHE_PORT) + + # Set artifact module path + # TODO: restrict this to workflow jobs, or JTs expecting artifacts + env['ANSIBLE_LIBRARY'] = self.get_path_to('..', 'plugins', 'library') + return env def build_args(self, job, **kwargs): @@ -893,7 +898,7 @@ class RunJob(BaseTask): 'tower_user_name': job.created_by.username, }) if job.extra_vars_dict: - if kwargs.get('display', False) and job.job_template and job.job_template.survey_enabled: + if kwargs.get('display', False) and job.job_template: extra_vars.update(json.loads(job.display_extra_vars())) else: extra_vars.update(job.extra_vars_dict) diff --git a/awx/main/tests/functional/models/test_workflow.py b/awx/main/tests/functional/models/test_workflow.py index 48b0fecaf5..cc61c34ed9 100644 --- a/awx/main/tests/functional/models/test_workflow.py +++ b/awx/main/tests/functional/models/test_workflow.py @@ -3,8 +3,11 @@ import pytest # AWX -from awx.main.models.workflow import WorkflowJob, WorkflowJobTemplateNode +from awx.main.models.workflow import WorkflowJob, WorkflowJobNode, WorkflowJobTemplateNode +from awx.main.models.jobs import Job +from awx.main.models.projects import ProjectUpdate +@pytest.mark.django_db class TestWorkflowJob: @pytest.fixture def workflow_job(self, workflow_job_template_factory): @@ -21,7 +24,6 @@ class TestWorkflowJob: return wfj - @pytest.mark.django_db def test_inherit_job_template_workflow_nodes(self, mocker, workflow_job): workflow_job.inherit_job_template_workflow_nodes() @@ -31,4 +33,60 @@ class TestWorkflowJob: assert nodes[0].failure_nodes.filter(id=nodes[3].id).exists() assert nodes[3].failure_nodes.filter(id=nodes[4].id).exists() + def test_inherit_ancestor_artifacts_from_job(self, project, mocker): + """ + Assure that nodes along the line of execution inherit artifacts + from both jobs ran, and from the accumulation of old jobs + """ + # Related resources + wfj = WorkflowJob.objects.create(name='test-wf-job') + job = Job.objects.create(name='test-job', artifacts={'b': 43}) + # Workflow job nodes + job_node = WorkflowJobNode.objects.create(workflow_job=wfj, job=job, + ancestor_artifacts={'a': 42}) + queued_node = WorkflowJobNode.objects.create(workflow_job=wfj) + # Connect old job -> new job + mocker.patch.object(queued_node, 'get_parent_nodes', lambda: [job_node]) + assert queued_node.get_job_kwargs()['extra_vars'] == {'a': 42, 'b': 43} + assert queued_node.ancestor_artifacts == {'a': 42, 'b': 43} + + def test_inherit_ancestor_artifacts_from_project_update(self, project, mocker): + """ + Test that the existence of a project update (no artifacts) does + not break the flow of ancestor_artifacts + """ + # Related resources + wfj = WorkflowJob.objects.create(name='test-wf-job') + update = ProjectUpdate.objects.create(name='test-update', project=project) + # Workflow job nodes + project_node = WorkflowJobNode.objects.create(workflow_job=wfj, job=update, + ancestor_artifacts={'a': 42, 'b': 43}) + queued_node = WorkflowJobNode.objects.create(workflow_job=wfj) + # Connect project update -> new job + mocker.patch.object(queued_node, 'get_parent_nodes', lambda: [project_node]) + assert queued_node.get_job_kwargs()['extra_vars'] == {'a': 42, 'b': 43} + assert queued_node.ancestor_artifacts == {'a': 42, 'b': 43} + +@pytest.mark.django_db +class TestWorkflowJobTemplate: + + @pytest.fixture + def wfjt(self, workflow_job_template_factory): + wfjt = workflow_job_template_factory('test').workflow_job_template + nodes = [WorkflowJobTemplateNode.objects.create(workflow_job_template=wfjt) for i in range(0, 3)] + nodes[0].success_nodes.add(nodes[1]) + nodes[1].failure_nodes.add(nodes[2]) + return wfjt + + def test_node_parentage(self, wfjt): + # test success parent + wfjt_node = wfjt.workflow_job_template_nodes.all()[1] + parent_qs = wfjt_node.get_parent_nodes() + assert len(parent_qs) == 1 + assert parent_qs[0] == wfjt.workflow_job_template_nodes.all()[0] + # test failure parent + wfjt_node = wfjt.workflow_job_template_nodes.all()[2] + parent_qs = wfjt_node.get_parent_nodes() + assert len(parent_qs) == 1 + assert parent_qs[0] == wfjt.workflow_job_template_nodes.all()[1] diff --git a/awx/main/tests/unit/models/test_workflow_unit.py b/awx/main/tests/unit/models/test_workflow_unit.py index 4445a758d9..9df61ffe97 100644 --- a/awx/main/tests/unit/models/test_workflow_unit.py +++ b/awx/main/tests/unit/models/test_workflow_unit.py @@ -6,6 +6,7 @@ from awx.main.models.workflow import ( WorkflowJobTemplate, WorkflowJobTemplateNode, WorkflowJobInheritNodesMixin, WorkflowJob, WorkflowJobNode ) +import mock class TestWorkflowJobInheritNodesMixin(): class TestCreateWorkflowJobNodes(): @@ -151,6 +152,7 @@ class TestWorkflowJobCreate: unified_job_template=wfjt_node_with_prompts.unified_job_template, workflow_job=workflow_job_unit) +@mock.patch('awx.main.models.workflow.WorkflowNodeBase.get_parent_nodes', lambda self: []) class TestWorkflowJobNodeJobKWARGS: """ Tests for building the keyword arguments that go into creating and diff --git a/awx/plugins/callback/job_event_callback.py b/awx/plugins/callback/job_event_callback.py index 1f0e41797d..67f36612f6 100644 --- a/awx/plugins/callback/job_event_callback.py +++ b/awx/plugins/callback/job_event_callback.py @@ -184,6 +184,9 @@ class BaseCallbackModule(object): if getattr(self, 'ad_hoc_command_id', None): msg['ad_hoc_command_id'] = self.ad_hoc_command_id + if getattr(self, 'artifact_data', None): + msg['artifact_data'] = self.artifact_data + active_pid = os.getpid() if self.job_callback_debug: msg.update({ @@ -416,6 +419,9 @@ class JobCallbackModule(BaseCallbackModule): event_data['task'] = task_name if role_name and event not in self.EVENTS_WITHOUT_TASK: event_data['role'] = role_name + self.artifact_data = None + if 'res' in event_data and 'artifact_data' in event_data['res']: + self.artifact_data = event_data['res']['artifact_data'] super(JobCallbackModule, self)._log_event(event, **event_data) def playbook_on_start(self): diff --git a/awx/plugins/library/set_artifact.py b/awx/plugins/library/set_artifact.py new file mode 100644 index 0000000000..680b4bda96 --- /dev/null +++ b/awx/plugins/library/set_artifact.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +from ansible.module_utils.basic import * # noqa + +DOCUMENTATION = ''' +--- +module: set_artifact +short_description: Stash some Ansible variables for later +description: + - Saves a user-specified JSON dictionary of variables from a playbook + for later use +version_added: "2.2" +options: +requirements: [ ] +author: Alan Rominger +''' + +EXAMPLES = ''' +# Example fact output: + +# Simple specifying of an artifact dictionary, will be passed on callback +- set_artifact: + data: + one_artifact: "{{ local_var * 2 }}" + another_artifact: "{{ some_registered_var.results | map(attribute='ansible_facts.some_fact') | list }}" + + +# Specifying a local path to save the artifacts to +- set_artifact: + data: + one_artifact: "{{ local_var * 2 }}" + another_artifact: "{{ some_registered_var.results | map(attribute='ansible_facts.some_fact') | list }}" + dest=/tmp/prefix-{{ inventory_hostname }} + + + +host | success >> { + "artifact_data": {} +} +''' + +def main(): + import json + module = AnsibleModule( + argument_spec = dict( + data=dict( + type='dict', + default={} + ) + ) + ) + results = dict( + changed=True, + artifact_data=json.dumps(module.params.get('data')) + ) + module.exit_json(**results) + +if __name__ == '__main__': + main() From ef861e525640a2da3aea25db8ef7137b23da6674 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 12 Oct 2016 17:53:39 -0400 Subject: [PATCH 14/21] move set_artifact module to its own repository --- awx/main/tasks.py | 5 --- awx/plugins/library/set_artifact.py | 59 ----------------------------- 2 files changed, 64 deletions(-) delete mode 100644 awx/plugins/library/set_artifact.py diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 2c1dc45538..6efe4309c4 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -821,11 +821,6 @@ class RunJob(BaseTask): env['ANSIBLE_CACHE_PLUGINS'] = self.get_path_to('..', 'plugins', 'fact_caching') env['ANSIBLE_CACHE_PLUGIN'] = "tower" env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] = "tcp://127.0.0.1:%s" % str(settings.FACT_CACHE_PORT) - - # Set artifact module path - # TODO: restrict this to workflow jobs, or JTs expecting artifacts - env['ANSIBLE_LIBRARY'] = self.get_path_to('..', 'plugins', 'library') - return env def build_args(self, job, **kwargs): diff --git a/awx/plugins/library/set_artifact.py b/awx/plugins/library/set_artifact.py deleted file mode 100644 index 680b4bda96..0000000000 --- a/awx/plugins/library/set_artifact.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python - -from ansible.module_utils.basic import * # noqa - -DOCUMENTATION = ''' ---- -module: set_artifact -short_description: Stash some Ansible variables for later -description: - - Saves a user-specified JSON dictionary of variables from a playbook - for later use -version_added: "2.2" -options: -requirements: [ ] -author: Alan Rominger -''' - -EXAMPLES = ''' -# Example fact output: - -# Simple specifying of an artifact dictionary, will be passed on callback -- set_artifact: - data: - one_artifact: "{{ local_var * 2 }}" - another_artifact: "{{ some_registered_var.results | map(attribute='ansible_facts.some_fact') | list }}" - - -# Specifying a local path to save the artifacts to -- set_artifact: - data: - one_artifact: "{{ local_var * 2 }}" - another_artifact: "{{ some_registered_var.results | map(attribute='ansible_facts.some_fact') | list }}" - dest=/tmp/prefix-{{ inventory_hostname }} - - - -host | success >> { - "artifact_data": {} -} -''' - -def main(): - import json - module = AnsibleModule( - argument_spec = dict( - data=dict( - type='dict', - default={} - ) - ) - ) - results = dict( - changed=True, - artifact_data=json.dumps(module.params.get('data')) - ) - module.exit_json(**results) - -if __name__ == '__main__': - main() From 638b07e17bb66e801a48dc1de44304d83ad3d459 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 13 Oct 2016 10:47:11 -0400 Subject: [PATCH 15/21] remove yaml and json imports from workflow --- awx/main/models/workflow.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 3c175be064..9ec728ed9a 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -24,8 +24,6 @@ from awx.main.models.mixins import ResourceMixin from awx.main.redact import REPLACE_STR from copy import copy -import yaml -import json __all__ = ['WorkflowJobTemplate', 'WorkflowJob', 'WorkflowJobOptions', 'WorkflowJobNode', 'WorkflowJobTemplateNode',] From 7472fbae8f5ba6695f269c58fcc1c66b4e2ca576 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 13 Oct 2016 11:05:06 -0400 Subject: [PATCH 16/21] insert back in REPLACE_STR import that was lost --- awx/main/models/unified_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index d40f50cca3..8c953c61e6 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -33,7 +33,7 @@ from djcelery.models import TaskMeta from awx.main.models.base import * # noqa from awx.main.models.schedules import Schedule from awx.main.utils import decrypt_field, _inventory_updates -from awx.main.redact import UriCleaner +from awx.main.redact import UriCleaner, REPLACE_STR from awx.main.consumers import emit_channel_notification __all__ = ['UnifiedJobTemplate', 'UnifiedJob'] From 4220246400e658e814c1e4f149e7e8ac7d3024fb Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Fri, 14 Oct 2016 01:43:14 -0400 Subject: [PATCH 17/21] use uwsgi/daphne/nginx for dev --- Makefile | 20 +++++++++++++++++++- Procfile | 4 +++- requirements/requirements.txt | 3 ++- tools/docker-compose.yml | 10 +++++++++- tools/docker-compose/Dockerfile | 2 +- 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index bb4d3021c0..f6b26e8403 100644 --- a/Makefile +++ b/Makefile @@ -387,6 +387,24 @@ flower: fi; \ $(PYTHON) manage.py celery flower --address=0.0.0.0 --port=5555 --broker=amqp://guest:guest@$(RABBITMQ_HOST):5672// +uwsgi: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ + uwsgi --socket :8050 --module=awx.wsgi:application --home=/venv/tower --chdir=/tower_devel/ --vacuum --processes=5 --harakiri=60 --static-map /static=/tower_devel/awx/public/static + +daphne: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ + daphne -b 127.0.0.1 -p 8051 awx.asgi:channel_layer + +runworker: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ + $(PYTHON) manage.py runworker --only-channels websocket.* + # Run the built-in development webserver (by default on http://localhost:8013). runserver: @if [ "$(VENV_BASE)" ]; then \ @@ -753,7 +771,7 @@ docker-auth: # Docker Compose Development environment docker-compose: docker-auth - TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose.yml up --no-recreate + TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose.yml up --no-recreate nginx tower docker-compose-cluster: docker-auth TAG=$(COMPOSE_TAG) docker-compose -f tools/docker-compose-cluster.yml up diff --git a/Procfile b/Procfile index b63680a2e2..09dfe2411c 100644 --- a/Procfile +++ b/Procfile @@ -1,5 +1,7 @@ -runserver: make runserver +runworker: make runworker +daphne: make daphne celeryd: make celeryd receiver: make receiver factcacher: make factcacher flower: make flower +uwsgi: make uwsgi diff --git a/requirements/requirements.txt b/requirements/requirements.txt index ce9bf8b6c4..ae773ddf78 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -134,4 +134,5 @@ wsgiref==0.1.2 xmltodict==0.9.2 channels==0.17.2 asgi_amqp==0.3 - +uwsgi==2.0.14 +daphne==0.15.0 diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index 2e5b18460d..06dede5fbc 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -11,8 +11,9 @@ services: RABBITMQ_VHOST: / ports: - "8080:8080" - - "8013:8013" - "5555:5555" + - "8050:8050" + - "8051:8051" links: - postgres - memcached @@ -34,6 +35,13 @@ services: ports: - "15672:15672" + nginx: + image: gcr.io/ansible-tower-engineering/tower_nginx:${TAG} + ports: + - "8013:80" + links: + - tower + # Source Code Synchronization Container # sync: # build: diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile index efa3f7709c..6a3bec6c38 100644 --- a/tools/docker-compose/Dockerfile +++ b/tools/docker-compose/Dockerfile @@ -27,6 +27,6 @@ RUN ln -s /tower_devel/tools/docker-compose/start_development.sh /start_developm WORKDIR /tmp RUN SWIG_FEATURES="-cpperraswarn -includeall -D__`uname -m`__ -I/usr/include/openssl" VENV_BASE="/venv" make requirements_dev WORKDIR / -EXPOSE 8013 8080 22 +EXPOSE 8050 8051 8080 22 ENTRYPOINT ["/usr/bin/dumb-init"] CMD /start_development.sh From eb3ec0b0837f845c230ec743ddd796fb830829d2 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Fri, 14 Oct 2016 02:00:56 -0400 Subject: [PATCH 18/21] daphne should listen on 0.0.0.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f6b26e8403..54818ad6eb 100644 --- a/Makefile +++ b/Makefile @@ -397,7 +397,7 @@ daphne: @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/tower/bin/activate; \ fi; \ - daphne -b 127.0.0.1 -p 8051 awx.asgi:channel_layer + daphne -b 0.0.0.0 -p 8051 awx.asgi:channel_layer runworker: @if [ "$(VENV_BASE)" ]; then \ From 90ed95abd1b92c9cd6c40cc146b45c114586f573 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Fri, 14 Oct 2016 02:23:12 -0400 Subject: [PATCH 19/21] add https ports --- tools/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index 06dede5fbc..f68d5d594c 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -38,6 +38,7 @@ services: nginx: image: gcr.io/ansible-tower-engineering/tower_nginx:${TAG} ports: + - "8043:443" - "8013:80" links: - tower From 8d89d68d5e45d6c3f42b645d4ba82fe9dee890e0 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 14 Oct 2016 08:54:37 -0400 Subject: [PATCH 20/21] flake8 fixes for Jenkins --- awx/asgi.py | 4 +-- .../management/commands/deprovision_node.py | 2 +- .../management/commands/register_instance.py | 2 +- .../tests/functional/api/test_settings.py | 1 - awx/settings/defaults.py | 29 +++++++++---------- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/awx/asgi.py b/awx/asgi.py index 42d800d939..3190a7032c 100644 --- a/awx/asgi.py +++ b/awx/asgi.py @@ -6,9 +6,10 @@ from awx import __version__ as tower_version # Prepare the AWX environment. from awx import prepare_env, MODE -prepare_env() +prepare_env() # NOQA from django.core.wsgi import get_wsgi_application # NOQA +from channels.asgi import get_channel_layer """ ASGI config for AWX project. @@ -29,7 +30,6 @@ if MODE == 'production': logger.error("Missing or incorrect metadata for Tower version. Ensure Tower was installed using the setup playbook.") raise Exception("Missing or incorrect metadata for Tower version. Ensure Tower was installed using the setup playbook.") -from channels.asgi import get_channel_layer os.environ.setdefault("DJANGO_SETTINGS_MODULE", "awx.settings") diff --git a/awx/main/management/commands/deprovision_node.py b/awx/main/management/commands/deprovision_node.py index ac27960daa..52b9e4f115 100644 --- a/awx/main/management/commands/deprovision_node.py +++ b/awx/main/management/commands/deprovision_node.py @@ -1,7 +1,7 @@ # Copyright (c) 2016 Ansible, Inc. # All Rights Reserved -from django.core.management.base import CommandError, BaseCommand +from django.core.management.base import BaseCommand from optparse import make_option from awx.main.models import Instance diff --git a/awx/main/management/commands/register_instance.py b/awx/main/management/commands/register_instance.py index 04b2b08a8a..01e2011aa0 100644 --- a/awx/main/management/commands/register_instance.py +++ b/awx/main/management/commands/register_instance.py @@ -5,7 +5,7 @@ from awx.main.models import Instance from django.conf import settings from optparse import make_option -from django.core.management.base import CommandError, BaseCommand +from django.core.management.base import BaseCommand class Command(BaseCommand): """ diff --git a/awx/main/tests/functional/api/test_settings.py b/awx/main/tests/functional/api/test_settings.py index 51314defb5..450fbd8dc9 100644 --- a/awx/main/tests/functional/api/test_settings.py +++ b/awx/main/tests/functional/api/test_settings.py @@ -3,7 +3,6 @@ # Python import pytest -import mock # Django from django.core.urlresolvers import reverse diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index c740bed39b..630b5ee6ee 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -361,21 +361,20 @@ CELERY_QUEUES = ( Broadcast('projects'), ) CELERY_ROUTES = {'awx.main.tasks.run_job': {'queue': 'jobs', - 'routing_key': 'jobs'}, - 'awx.main.tasks.run_project_update': {'queue': 'projects'}, - 'awx.main.tasks.run_inventory_update': {'queue': 'jobs', - 'routing_key': 'jobs'}, - 'awx.main.tasks.run_ad_hoc_command': {'queue': 'jobs', - 'routing_key': 'jobs'}, - 'awx.main.tasks.run_system_job': {'queue': 'jobs', - 'routing_key': 'jobs'}, - 'awx.main.scheduler.tasks.run_job_launch': {'queue': 'scheduler', - 'routing_key': 'scheduler.job.launch'}, - 'awx.main.scheduler.tasks.run_job_complete': {'queue': 'scheduler', - 'routing_key': 'scheduler.job.complete'}, - 'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default', - 'routing_key': 'cluster.heartbeat'}, -} + 'routing_key': 'jobs'}, + 'awx.main.tasks.run_project_update': {'queue': 'projects'}, + 'awx.main.tasks.run_inventory_update': {'queue': 'jobs', + 'routing_key': 'jobs'}, + 'awx.main.tasks.run_ad_hoc_command': {'queue': 'jobs', + 'routing_key': 'jobs'}, + 'awx.main.tasks.run_system_job': {'queue': 'jobs', + 'routing_key': 'jobs'}, + 'awx.main.scheduler.tasks.run_job_launch': {'queue': 'scheduler', + 'routing_key': 'scheduler.job.launch'}, + 'awx.main.scheduler.tasks.run_job_complete': {'queue': 'scheduler', + 'routing_key': 'scheduler.job.complete'}, + 'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default', + 'routing_key': 'cluster.heartbeat'},} CELERYBEAT_SCHEDULE = { 'tower_scheduler': { From 0666e40be8777a0cc200f5e4aec9f3d9e932df9a Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Fri, 14 Oct 2016 12:04:30 -0400 Subject: [PATCH 21/21] fix make server to run other services --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 54818ad6eb..4f5a4fbe4b 100644 --- a/Makefile +++ b/Makefile @@ -357,12 +357,16 @@ dbshell: sudo -u postgres psql -d awx-dev server_noattach: - tmux new-session -d -s tower 'exec make runserver' + tmux new-session -d -s tower 'exec make uwsgi' tmux rename-window 'Tower' tmux select-window -t tower:0 tmux split-window -v 'exec make celeryd' - tmux new-window 'exec make receiver' + tmux new-window 'exec make daphne' tmux select-window -t tower:1 + tmux rename-window 'WebSockets' + tmux split-window -h 'exec make runworker' + tmux new-window 'exec make receiver' + tmux select-window -t tower:2 tmux rename-window 'Extra Services' tmux split-window -h 'exec make factcacher'