From 31650bb0bd99ecf689f08497faa935ffd47ab869 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Fri, 22 Nov 2019 11:39:08 -0500 Subject: [PATCH 01/36] Remove usage of idle_timeout when checking status of isolated / containerized jobs --- awx/main/isolated/manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/awx/main/isolated/manager.py b/awx/main/isolated/manager.py index 642ba373a4..ae09176cbc 100644 --- a/awx/main/isolated/manager.py +++ b/awx/main/isolated/manager.py @@ -40,7 +40,6 @@ class IsolatedManager(object): """ self.cancelled_callback = cancelled_callback self.check_callback = check_callback - self.idle_timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT) self.started_at = None self.captured_command_artifact = False self.instance = None @@ -108,7 +107,6 @@ class IsolatedManager(object): 'verbosity': verbosity, 'cancel_callback': self.cancelled_callback, 'settings': { - 'idle_timeout': self.idle_timeout, 'job_timeout': settings.AWX_ISOLATED_LAUNCH_TIMEOUT, 'pexpect_timeout': getattr(settings, 'PEXPECT_TIMEOUT', 5), 'suppress_ansible_output': True, @@ -118,7 +116,7 @@ class IsolatedManager(object): def path_to(self, *args): return os.path.join(self.private_data_dir, *args) - def run_management_playbook(self, playbook, private_data_dir, **kw): + def run_management_playbook(self, playbook, private_data_dir, idle_timeout=None, **kw): iso_dir = tempfile.mkdtemp( prefix=playbook, dir=private_data_dir @@ -126,6 +124,8 @@ class IsolatedManager(object): params = self.runner_params.copy() params['playbook'] = playbook params['private_data_dir'] = iso_dir + if idle_timeout: + params['settings']['idle_timeout'] = idle_timeout params.update(**kw) if all([ getattr(settings, 'AWX_ISOLATED_KEY_GENERATION', False) is True, @@ -177,6 +177,7 @@ class IsolatedManager(object): logger.debug('Starting job {} on isolated host with `run_isolated.yml` playbook.'.format(self.instance.id)) runner_obj = self.run_management_playbook('run_isolated.yml', self.private_data_dir, + idle_timeout=max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT), extravars=extravars) if runner_obj.status == 'failed': From ee8775a08d96081c875d41f62daaa155b54957bf Mon Sep 17 00:00:00 2001 From: mabashian Date: Thu, 21 Nov 2019 11:05:06 -0500 Subject: [PATCH 02/36] Adds link to docs on container groups add/edit forms to match instance groups. Updates instance groups link. --- .../add-edit/add-instance-group.controller.js | 4 ++-- .../add-edit/edit-instance-group.controller.js | 4 ++-- .../container-groups/add-container-group.controller.js | 5 +++++ .../container-groups/add-container-group.view.html | 6 +++++- .../container-groups/edit-container-group.controller.js | 5 +++++ .../client/src/instance-groups/instance-groups.strings.js | 3 ++- 6 files changed, 21 insertions(+), 6 deletions(-) diff --git a/awx/ui/client/src/instance-groups/add-edit/add-instance-group.controller.js b/awx/ui/client/src/instance-groups/add-edit/add-instance-group.controller.js index 537096996b..d556c6dbbe 100644 --- a/awx/ui/client/src/instance-groups/add-edit/add-instance-group.controller.js +++ b/awx/ui/client/src/instance-groups/add-edit/add-instance-group.controller.js @@ -7,8 +7,8 @@ function AddController ($state, models, strings) { vm.panelTitle = strings.get('state.ADD_BREADCRUMB_LABEL'); vm.docs = { - url: 'https://docs.ansible.com/ansible-tower/latest/html/administration/clustering.html', - help_text: vm.strings.get('tooltips.DOCS_HELP_TEXT') + url: 'https://docs.ansible.com/ansible-tower/latest/html/userguide/instance_groups.html', + help_text: vm.strings.get('tooltips.IG_DOCS_HELP_TEXT') }; vm.tab = { diff --git a/awx/ui/client/src/instance-groups/add-edit/edit-instance-group.controller.js b/awx/ui/client/src/instance-groups/add-edit/edit-instance-group.controller.js index 18401ca08e..55972e36fe 100644 --- a/awx/ui/client/src/instance-groups/add-edit/edit-instance-group.controller.js +++ b/awx/ui/client/src/instance-groups/add-edit/edit-instance-group.controller.js @@ -17,8 +17,8 @@ function EditController ($rootScope, $state, models, strings) { vm.panelTitle = instanceGroup.get('name'); vm.docs = { - url: 'https://docs.ansible.com/ansible-tower/latest/html/administration/clustering.html', - help_text: vm.strings.get('tooltips.DOCS_HELP_TEXT') + url: 'https://docs.ansible.com/ansible-tower/latest/html/userguide/instance_groups.html', + help_text: vm.strings.get('tooltips.IG_DOCS_HELP_TEXT') }; vm.tab = { diff --git a/awx/ui/client/src/instance-groups/container-groups/add-container-group.controller.js b/awx/ui/client/src/instance-groups/container-groups/add-container-group.controller.js index fc6bfaf846..cdab0e89fc 100644 --- a/awx/ui/client/src/instance-groups/container-groups/add-container-group.controller.js +++ b/awx/ui/client/src/instance-groups/container-groups/add-container-group.controller.js @@ -10,6 +10,11 @@ function AddContainerGroupController(ToJSON, $scope, $state, models, strings, i1 vm.panelTitle = strings.get('state.ADD_CONTAINER_GROUP_BREADCRUMB_LABEL'); vm.lookUpTitle = strings.get('container.LOOK_UP_TITLE'); + vm.docs = { + url: 'https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#ag-container-groups', + help_text: vm.strings.get('tooltips.CG_DOCS_HELP_TEXT') + }; + vm.form = instanceGroup.createFormSchema('post'); vm.form.name.required = true; delete vm.form.name.help_text; diff --git a/awx/ui/client/src/instance-groups/container-groups/add-container-group.view.html b/awx/ui/client/src/instance-groups/container-groups/add-container-group.view.html index a7c3dcb446..3e27ee18bf 100644 --- a/awx/ui/client/src/instance-groups/container-groups/add-container-group.view.html +++ b/awx/ui/client/src/instance-groups/container-groups/add-container-group.view.html @@ -6,7 +6,11 @@ - + + + + + {{:: vm.strings.get('tab.DETAILS') }} {{:: vm.strings.get('tab.JOBS') }} diff --git a/awx/ui/client/src/instance-groups/container-groups/edit-container-group.controller.js b/awx/ui/client/src/instance-groups/container-groups/edit-container-group.controller.js index 62740a559d..45b6f9a3b1 100644 --- a/awx/ui/client/src/instance-groups/container-groups/edit-container-group.controller.js +++ b/awx/ui/client/src/instance-groups/container-groups/edit-container-group.controller.js @@ -23,6 +23,11 @@ function EditContainerGroupController($rootScope, $scope, $state, models, string vm.panelTitle = EditContainerGroupDataset.data.name; vm.lookUpTitle = strings.get('container.LOOK_UP_TITLE'); + vm.docs = { + url: 'https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#ag-container-groups', + help_text: vm.strings.get('tooltips.CG_DOCS_HELP_TEXT') + }; + vm.form = instanceGroup.createFormSchema('post'); vm.switchDisabled = false; vm.form.disabled = !instanceGroup.has('options', 'actions.PUT'); diff --git a/awx/ui/client/src/instance-groups/instance-groups.strings.js b/awx/ui/client/src/instance-groups/instance-groups.strings.js index 6e1e118f08..74fd210350 100644 --- a/awx/ui/client/src/instance-groups/instance-groups.strings.js +++ b/awx/ui/client/src/instance-groups/instance-groups.strings.js @@ -32,7 +32,8 @@ function InstanceGroupsStrings(BaseString) { ns.tooltips = { ADD_INSTANCE_GROUP: t.s('Create a new Instance Group'), ASSOCIATE_INSTANCES: t.s('Associate an existing Instance'), - DOCS_HELP_TEXT: t.s('Instance Groups Help') + IG_DOCS_HELP_TEXT: t.s('Instance Groups Help'), + CG_DOCS_HELP_TEXT: t.s('Container Groups Help') }; ns.instance = { From 12363ae1753eddf33e00a7d9526daa2896a6bbeb Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 25 Nov 2019 11:32:41 -0500 Subject: [PATCH 03/36] Fix Docker build caching The flow will need to be: - Pre-pull image you want to use - Re-tag as image:$(COMPOSE_TAG) - COMPOSE_TAG=mytag make docker-compose-build --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 98caaba7df..03f6f21941 100644 --- a/Makefile +++ b/Makefile @@ -654,7 +654,6 @@ docker-compose-build: awx-devel-build # Base development image build awx-devel-build: docker build -t ansible/awx_devel -f tools/docker-compose/Dockerfile \ - --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:devel \ --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) . docker tag ansible/awx_devel $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) #docker push $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) From ce8c0066d048bf4d91a53a51ebbbd6dc9a241b64 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 25 Nov 2019 12:51:49 -0500 Subject: [PATCH 04/36] Fix downstream tests I backported how we do the VERSION detection in 3.5.something. This should already be fixed upstream. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 03f6f21941..8f9716e64b 100644 --- a/Makefile +++ b/Makefile @@ -364,7 +364,7 @@ check: flake8 pep8 # pyflakes pylint awx-link: cp -R /tmp/awx.egg-info /awx_devel/ || true - sed -i "s/placeholder/$(shell git describe --long | sed 's/\./\\./g')/" /awx_devel/awx.egg-info/PKG-INFO + sed -i "s/placeholder/$(shell cat VERSION)/" /awx_devel/awx.egg-info/PKG-INFO cp -f /tmp/awx.egg-link /venv/awx/lib/python$(PYTHON_VERSION)/site-packages/awx.egg-link TEST_DIRS ?= awx/main/tests/unit awx/main/tests/functional awx/conf/tests awx/sso/tests From 695eab1fdd5856c74d71a075ef97b78d0fd6984f Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 26 Nov 2019 09:39:05 -0500 Subject: [PATCH 05/36] fix duplicate exception sanity error --- awx_collection/plugins/modules/tower_credential.py | 2 +- awx_collection/plugins/modules/tower_group.py | 2 +- awx_collection/plugins/modules/tower_label.py | 2 +- awx_collection/plugins/modules/tower_team.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/awx_collection/plugins/modules/tower_credential.py b/awx_collection/plugins/modules/tower_credential.py index a87d76fecb..d2f2d9bd5f 100644 --- a/awx_collection/plugins/modules/tower_credential.py +++ b/awx_collection/plugins/modules/tower_credential.py @@ -404,7 +404,7 @@ def main(): result = credential.delete(**params) except (exc.NotFound) as excinfo: module.fail_json(msg='Failed to update credential, organization not found: {0}'.format(excinfo), changed=False) - except (exc.ConnectionError, exc.BadRequest, exc.NotFound, exc.AuthError) as excinfo: + except (exc.ConnectionError, exc.BadRequest, exc.AuthError) as excinfo: module.fail_json(msg='Failed to update credential: {0}'.format(excinfo), changed=False) json_output['changed'] = result['changed'] diff --git a/awx_collection/plugins/modules/tower_group.py b/awx_collection/plugins/modules/tower_group.py index bd9feaf91f..0b4269b5f7 100644 --- a/awx_collection/plugins/modules/tower_group.py +++ b/awx_collection/plugins/modules/tower_group.py @@ -180,7 +180,7 @@ def main(): result = group.delete(**params) except (exc.NotFound) as excinfo: module.fail_json(msg='Failed to update the group, inventory not found: {0}'.format(excinfo), changed=False) - except (exc.ConnectionError, exc.BadRequest, exc.NotFound, exc.AuthError) as excinfo: + except (exc.ConnectionError, exc.BadRequest, exc.AuthError) as excinfo: module.fail_json(msg='Failed to update the group: {0}'.format(excinfo), changed=False) json_output['changed'] = result['changed'] diff --git a/awx_collection/plugins/modules/tower_label.py b/awx_collection/plugins/modules/tower_label.py index 254d9d4986..e5085cfd27 100644 --- a/awx_collection/plugins/modules/tower_label.py +++ b/awx_collection/plugins/modules/tower_label.py @@ -94,7 +94,7 @@ def main(): result = label.delete(name=name, organization=org['id']) except (exc.NotFound) as excinfo: module.fail_json(msg='Failed to update label, organization not found: {0}'.format(excinfo), changed=False) - except (exc.ConnectionError, exc.BadRequest, exc.NotFound, exc.AuthError) as excinfo: + except (exc.ConnectionError, exc.BadRequest, exc.AuthError) as excinfo: module.fail_json(msg='Failed to update label: {0}'.format(excinfo), changed=False) json_output['changed'] = result['changed'] diff --git a/awx_collection/plugins/modules/tower_team.py b/awx_collection/plugins/modules/tower_team.py index 80aff36f04..dc34f4d706 100644 --- a/awx_collection/plugins/modules/tower_team.py +++ b/awx_collection/plugins/modules/tower_team.py @@ -103,7 +103,7 @@ def main(): result = team.delete(name=name, organization=org['id']) except (exc.NotFound) as excinfo: module.fail_json(msg='Failed to update team, organization not found: {0}'.format(excinfo), changed=False) - except (exc.ConnectionError, exc.BadRequest, exc.NotFound, exc.AuthError) as excinfo: + except (exc.ConnectionError, exc.BadRequest, exc.AuthError) as excinfo: module.fail_json(msg='Failed to update team: {0}'.format(excinfo), changed=False) json_output['changed'] = result['changed'] From 68f17eb370b114f837d627793f7cde2c43e15d09 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 26 Nov 2019 23:29:50 -0500 Subject: [PATCH 06/36] bump asgi-amqp dependency --- requirements/requirements.in | 2 +- requirements/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index e26d228bd4..eb61733474 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,6 +1,6 @@ ansible-runner==1.4.4 appdirs==1.4.2 -asgi-amqp==1.1.3 +asgi-amqp==1.1.4 azure-keyvault==1.1.0 boto==2.47.0 channels==1.1.8 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 89d1ef12ac..e446994ee9 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,7 +3,7 @@ amqp==2.4.2 # via kombu ansible-runner==1.4.4 appdirs==1.4.2 argparse==1.4.0 # via uwsgitop -asgi-amqp==1.1.3 +asgi-amqp==1.1.4 asgiref==1.1.2 # via asgi-amqp, channels, daphne asn1crypto==0.24.0 # via cryptography attrs==19.1.0 # via automat, service-identity, twisted From ffdcb2f8eb7e2dc43aa0d05dd8a5d817d28da759 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 19 Nov 2019 09:21:04 -0500 Subject: [PATCH 07/36] fix busted tests --- awx/main/tests/functional/models/test_notifications.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/awx/main/tests/functional/models/test_notifications.py b/awx/main/tests/functional/models/test_notifications.py index 1b671efdcb..00cc217a82 100644 --- a/awx/main/tests/functional/models/test_notifications.py +++ b/awx/main/tests/functional/models/test_notifications.py @@ -88,6 +88,9 @@ class TestJobNotificationMixin(object): 'verbosity': int}, 'job_friendly_name': str, 'job_metadata': str, + 'approval_status': str, + 'approval_node_name': str, + 'workflow_url': str, 'url': str} From ea5d4293995b4eadc517ee75d3b85eed46c3ed29 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 25 Nov 2019 11:50:09 -0500 Subject: [PATCH 08/36] fix a few bugs with the session and oauth2 cleanup scheduled jobs see: https://github.com/ansible/tower/issues/3940 --- awx/api/serializers.py | 4 +++ awx/main/models/jobs.py | 25 ++++++++++--------- awx/main/tasks.py | 9 ++++--- .../tests/unit/models/test_system_jobs.py | 12 ++++++--- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index a3d8d43306..d95a754e71 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -4658,6 +4658,10 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria def get_summary_fields(self, obj): summary_fields = super(ScheduleSerializer, self).get_summary_fields(obj) + + if isinstance(obj.unified_job_template, SystemJobTemplate): + summary_fields['unified_job_template']['job_type'] = obj.unified_job_template.job_type + if 'inventory' in summary_fields: return summary_fields diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index d573d1ed96..ae2f772526 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -1179,18 +1179,19 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions): for key in unallowed_vars: rejected[key] = data.pop(key) - if 'days' in data: - try: - if type(data['days']) is bool: - raise ValueError - if float(data['days']) != int(data['days']): - raise ValueError - days = int(data['days']) - if days < 0: - raise ValueError - except ValueError: - errors_list.append(_("days must be a positive integer.")) - rejected['days'] = data.pop('days') + if self.job_type in ('cleanup_jobs', 'cleanup_activitystream'): + if 'days' in data: + try: + if isinstance(data['days'], (bool, type(None))): + raise ValueError + if float(data['days']) != int(data['days']): + raise ValueError + days = int(data['days']) + if days < 0: + raise ValueError + except ValueError: + errors_list.append(_("days must be a positive integer.")) + rejected['days'] = data.pop('days') if errors_list: errors['extra_vars'] = errors_list diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 7429f8f458..37ef703a29 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2761,10 +2761,11 @@ class RunSystemJob(BaseTask): json_vars = {} else: json_vars = json.loads(system_job.extra_vars) - if 'days' in json_vars: - args.extend(['--days', str(json_vars.get('days', 60))]) - if 'dry_run' in json_vars and json_vars['dry_run']: - args.extend(['--dry-run']) + if system_job.job_type in ('cleanup_jobs', 'cleanup_activitystream'): + if 'days' in json_vars: + args.extend(['--days', str(json_vars.get('days', 60))]) + if 'dry_run' in json_vars and json_vars['dry_run']: + args.extend(['--dry-run']) if system_job.job_type == 'cleanup_jobs': args.extend(['--jobs', '--project-updates', '--inventory-updates', '--management-jobs', '--ad-hoc-commands', '--workflow-jobs', diff --git a/awx/main/tests/unit/models/test_system_jobs.py b/awx/main/tests/unit/models/test_system_jobs.py index 045928be07..2ed9204adb 100644 --- a/awx/main/tests/unit/models/test_system_jobs.py +++ b/awx/main/tests/unit/models/test_system_jobs.py @@ -12,7 +12,9 @@ from awx.main.models import SystemJobTemplate {"days": 13435}, ]) def test_valid__clean_extra_data_system_jobs(extra_data): - accepted, rejected, errors = SystemJobTemplate().accept_or_ignore_variables(extra_data) + accepted, rejected, errors = SystemJobTemplate( + job_type='cleanup_jobs' + ).accept_or_ignore_variables(extra_data) assert not rejected assert not errors @@ -32,12 +34,14 @@ def test_valid__clean_extra_data_system_jobs(extra_data): {"days": "foobar"}, ]) def test_invalid__extra_data_system_jobs(extra_data): - accepted, rejected, errors = SystemJobTemplate().accept_or_ignore_variables(extra_data) + accepted, rejected, errors = SystemJobTemplate( + job_type='cleanup_jobs' + ).accept_or_ignore_variables(extra_data) assert str(errors['extra_vars'][0]) == u'days must be a positive integer.' def test_unallowed_system_job_data(): - sjt = SystemJobTemplate() + sjt = SystemJobTemplate(job_type='cleanup_jobs') accepted, ignored, errors = sjt.accept_or_ignore_variables({ 'days': 34, 'foobar': 'baz' @@ -54,7 +58,7 @@ def test_reject_other_prommpts(): def test_reject_some_accept_some(): - sjt = SystemJobTemplate() + sjt = SystemJobTemplate(job_type='cleanup_jobs') accepted, ignored, errors = sjt._accept_or_ignore_job_kwargs(limit="", extra_vars={ 'days': 34, 'foobar': 'baz' From ee6e28e0661f1e3a6d55cf02dc6e6ef76543c4e1 Mon Sep 17 00:00:00 2001 From: mabashian Date: Tue, 26 Nov 2019 15:05:45 -0500 Subject: [PATCH 09/36] Only show the days to keep input on the scheduler for system jobs that require it. Hides this field for cleaning up tokens and sessions. --- awx/ui/client/src/management-jobs/card/card.controller.js | 1 - .../management-jobs/scheduler/schedulerForm.partial.html | 2 +- .../src/scheduler/factories/schedule-post.factory.js | 2 +- awx/ui/client/src/scheduler/schedulerAdd.controller.js | 5 ++++- awx/ui/client/src/scheduler/schedulerEdit.controller.js | 7 +++++-- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/awx/ui/client/src/management-jobs/card/card.controller.js b/awx/ui/client/src/management-jobs/card/card.controller.js index 930f04cd51..d266900ea5 100644 --- a/awx/ui/client/src/management-jobs/card/card.controller.js +++ b/awx/ui/client/src/management-jobs/card/card.controller.js @@ -31,7 +31,6 @@ export default }; getManagementJobs(); - $scope.cleanupJob = true; // This handles the case where the user refreshes the management job notifications page. if($state.current.name === 'managementJobsList.notifications') { $scope.activeCard = parseInt($state.params.management_id); diff --git a/awx/ui/client/src/management-jobs/scheduler/schedulerForm.partial.html b/awx/ui/client/src/management-jobs/scheduler/schedulerForm.partial.html index 065a78d4e3..58dfbff4c9 100644 --- a/awx/ui/client/src/management-jobs/scheduler/schedulerForm.partial.html +++ b/awx/ui/client/src/management-jobs/scheduler/schedulerForm.partial.html @@ -164,7 +164,7 @@ ng-show="sheduler_frequency_error"> -
+